mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-12 22:09:48 +08:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f8e434d2a0 | |||
| bb66585c00 | |||
| d6442e1ab4 | |||
| 2f09f48726 | |||
| fe26c2b639 |
@@ -6,7 +6,7 @@
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Skills shared by Baoyu for improving daily work efficiency",
|
||||
"version": "1.108.0"
|
||||
"version": "1.109.0"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
@@ -2,6 +2,18 @@
|
||||
|
||||
English | [中文](./CHANGELOG.zh.md)
|
||||
|
||||
## 1.109.0 - 2026-04-21
|
||||
|
||||
### Features
|
||||
- `baoyu-url-to-markdown`: vendor the `baoyu-fetch` runtime into `scripts/lib` and run it through a local `scripts/baoyu-fetch` CLI so published skill installs are self-contained
|
||||
|
||||
### Fixes
|
||||
- `baoyu-fetch`: extract playable X/Twitter video MP4 variants for single posts and X Articles, choosing the highest-bitrate MP4 and rendering article videos as `[video](...)`
|
||||
- `sync-clawhub`: publish from the shared release file list so extensionless CLI entrypoints, `bun.lock`, and vendored `scripts/lib` files are uploaded
|
||||
|
||||
### Maintenance
|
||||
- Upgrade `defuddle` to 0.17.0 and `jsdom` to 29.0.2; override `@xmldom/xmldom` to 0.8.13 to keep the Defuddle dependency chain vulnerability-free
|
||||
|
||||
## 1.108.0 - 2026-04-19
|
||||
|
||||
### Refactor
|
||||
|
||||
@@ -2,6 +2,18 @@
|
||||
|
||||
[English](./CHANGELOG.md) | 中文
|
||||
|
||||
## 1.109.0 - 2026-04-21
|
||||
|
||||
### 新功能
|
||||
- `baoyu-url-to-markdown`:将 `baoyu-fetch` 运行时代码 vendored 到 `scripts/lib`,并通过本地 `scripts/baoyu-fetch` CLI 调用,使发布后的技能安装不再依赖 `baoyu-fetch` npm 包
|
||||
|
||||
### 修复
|
||||
- `baoyu-fetch`:修复 X/Twitter 单条内容与 X Article 的视频解析,提取可播放的最高码率 MP4,并将文章视频渲染为 `[video](...)`
|
||||
- `sync-clawhub`:改用共享 release 文件清单发布,确保无扩展名 CLI 入口、`bun.lock` 和 vendored `scripts/lib` 文件都会被上传
|
||||
|
||||
### 维护
|
||||
- 将 `defuddle` 升级到 0.17.0、`jsdom` 升级到 29.0.2,并通过 override 将 `@xmldom/xmldom` 固定到 0.8.13,清除 Defuddle 依赖链上的漏洞提示
|
||||
|
||||
## 1.108.0 - 2026-04-19
|
||||
|
||||
### 重构
|
||||
|
||||
+4
-4
@@ -21,18 +21,18 @@ bash scripts/sync-clawhub.sh # sync all skills
|
||||
bash scripts/sync-clawhub.sh <skill> # sync one skill
|
||||
```
|
||||
|
||||
Release hooks are configured via `.releaserc.yml`. This repo does not stage a separate release directory: release prep verifies that skills depend on published npm package versions, and publish reads the skill directory directly.
|
||||
Release hooks are configured via `.releaserc.yml`. This repo does not stage a separate release directory: publish reads the skill directory directly and validates that local package references and CLI bin targets are self-contained.
|
||||
|
||||
## Shared Workspace Packages
|
||||
|
||||
`packages/` is the **only** source of truth for shared runtime code. Publish shared packages to npm and reference them from skill script `package.json` files with semver ranges. Do not vendor shared packages into `skills/*/scripts/vendor/`.
|
||||
`packages/` is the source of truth for shared runtime code. Most skills consume shared packages from npm with semver ranges. `baoyu-url-to-markdown` is the exception: it vendors the `baoyu-fetch` runtime into `skills/baoyu-url-to-markdown/scripts/lib/` so the published skill is self-contained and does not depend on the `baoyu-fetch` npm package.
|
||||
|
||||
Current packages:
|
||||
- `baoyu-chrome-cdp` (Chrome CDP utilities), consumed by 5 skills (`baoyu-danger-gemini-web`, `baoyu-danger-x-to-markdown`, `baoyu-post-to-wechat`, `baoyu-post-to-weibo`, `baoyu-post-to-x`)
|
||||
- `baoyu-md` (shared Markdown rendering and placeholder pipeline), consumed by 3 skills (`baoyu-markdown-to-html`, `baoyu-post-to-wechat`, `baoyu-post-to-weibo`)
|
||||
- `baoyu-fetch` (URL-to-Markdown CLI), consumed by 1 skill (`baoyu-url-to-markdown`)
|
||||
- `baoyu-fetch` (URL-to-Markdown CLI), vendored into 1 skill (`baoyu-url-to-markdown`)
|
||||
|
||||
**How it works**: npm packages are built from `packages/` and published to the public npm registry. Skills depend on those packages with `^<version>` specs. Release prep runs `node scripts/verify-shared-package-deps.mjs` so `file:` dependencies and vendored workspace packages cannot slip back in.
|
||||
**How it works**: npm packages are built from `packages/` and published to the public npm registry. Skills normally depend on those packages with `^<version>` specs. Release prep runs `node scripts/verify-shared-package-deps.mjs` so accidental `file:` dependencies cannot slip back in. For vendored skill runtimes, keep the copied code under the skill directory and run `node scripts/publish-skill.mjs --skill-dir <skill> --version <version> --dry-run` before publishing.
|
||||
|
||||
**Update workflow**:
|
||||
1. Edit package under `packages/`
|
||||
|
||||
Generated
+287
-158
@@ -22,18 +22,52 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/css-color": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz",
|
||||
"integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==",
|
||||
"version": "5.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz",
|
||||
"integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@csstools/css-calc": "^2.1.3",
|
||||
"@csstools/css-color-parser": "^3.0.9",
|
||||
"@csstools/css-parser-algorithms": "^3.0.4",
|
||||
"@csstools/css-tokenizer": "^3.0.3",
|
||||
"lru-cache": "^10.4.3"
|
||||
"@asamuzakjp/generational-cache": "^1.0.1",
|
||||
"@csstools/css-calc": "^3.2.0",
|
||||
"@csstools/css-color-parser": "^4.1.0",
|
||||
"@csstools/css-parser-algorithms": "^4.0.0",
|
||||
"@csstools/css-tokenizer": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/dom-selector": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz",
|
||||
"integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/generational-cache": "^1.0.1",
|
||||
"@asamuzakjp/nwsapi": "^2.3.9",
|
||||
"bidi-js": "^1.0.3",
|
||||
"css-tree": "^3.2.1",
|
||||
"is-potential-custom-element-name": "^1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/generational-cache": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz",
|
||||
"integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/nwsapi": {
|
||||
"version": "2.3.9",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz",
|
||||
"integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@babel/runtime": {
|
||||
"version": "7.29.2",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz",
|
||||
@@ -44,6 +78,18 @@
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@bramus/specificity": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz",
|
||||
"integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"css-tree": "^3.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"specificity": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@changesets/apply-release-plan": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@changesets/apply-release-plan/-/apply-release-plan-7.1.0.tgz",
|
||||
@@ -307,9 +353,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/color-helpers": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz",
|
||||
"integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==",
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz",
|
||||
"integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -322,13 +368,13 @@
|
||||
],
|
||||
"license": "MIT-0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-calc": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz",
|
||||
"integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==",
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz",
|
||||
"integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -341,17 +387,17 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
"@csstools/css-parser-algorithms": "^4.0.0",
|
||||
"@csstools/css-tokenizer": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-color-parser": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz",
|
||||
"integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==",
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz",
|
||||
"integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -364,21 +410,21 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@csstools/color-helpers": "^5.1.0",
|
||||
"@csstools/css-calc": "^2.1.4"
|
||||
"@csstools/color-helpers": "^6.0.2",
|
||||
"@csstools/css-calc": "^3.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-parser-algorithms": "^3.0.5",
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
"@csstools/css-parser-algorithms": "^4.0.0",
|
||||
"@csstools/css-tokenizer": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-parser-algorithms": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz",
|
||||
"integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==",
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz",
|
||||
"integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -391,16 +437,40 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@csstools/css-tokenizer": "^3.0.4"
|
||||
"@csstools/css-tokenizer": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-syntax-patches-for-csstree": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz",
|
||||
"integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/csstools"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/csstools"
|
||||
}
|
||||
],
|
||||
"license": "MIT-0",
|
||||
"peerDependencies": {
|
||||
"css-tree": "^3.2.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"css-tree": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-tokenizer": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz",
|
||||
"integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==",
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz",
|
||||
"integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -413,7 +483,7 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/runtime": {
|
||||
@@ -875,6 +945,23 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@exodus/bytes": {
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz",
|
||||
"integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@noble/hashes": "^1.8.0 || ^2.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@noble/hashes": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@img/colour": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz",
|
||||
@@ -1639,24 +1726,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@xmldom/xmldom": {
|
||||
"version": "0.8.11",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz",
|
||||
"integrity": "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==",
|
||||
"version": "0.8.13",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
|
||||
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "7.1.4",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
||||
"integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-colors": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz",
|
||||
@@ -1730,6 +1808,15 @@
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/bidi-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
|
||||
"integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"require-from-string": "^2.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
@@ -1892,6 +1979,19 @@
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/css-tree": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
|
||||
"integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mdn-data": "2.27.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/css-what": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
|
||||
@@ -1911,30 +2011,26 @@
|
||||
"devOptional": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cssstyle": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz",
|
||||
"integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==",
|
||||
"node_modules/data-urls": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz",
|
||||
"integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/css-color": "^3.2.0",
|
||||
"rrweb-cssom": "^0.8.0"
|
||||
"whatwg-mimetype": "^5.0.0",
|
||||
"whatwg-url": "^16.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"node_modules/data-urls/node_modules/whatwg-mimetype": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz",
|
||||
"integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz",
|
||||
"integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-mimetype": "^4.0.0",
|
||||
"whatwg-url": "^14.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
@@ -1974,9 +2070,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/defuddle": {
|
||||
"version": "0.14.0",
|
||||
"resolved": "https://registry.npmjs.org/defuddle/-/defuddle-0.14.0.tgz",
|
||||
"integrity": "sha512-btavZGd1WgiVqrVM62WGRXMUi/aU7ckTZiq0xXWLZMHvzIqNZjwIFQEDRx8MarD7fIgsB90NXZ9xHJkKtapt2Q==",
|
||||
"version": "0.17.0",
|
||||
"resolved": "https://registry.npmjs.org/defuddle/-/defuddle-0.17.0.tgz",
|
||||
"integrity": "sha512-rjRb9kxgrJLFhFJjXTSAitbmmSyvjEqGCi5JiyM6ImOf4leC/O+f7TPVPAymk/gnENFTrn9j7T/LTfW9IVwiLw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"commander": "^12.1.0"
|
||||
@@ -2418,15 +2514,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/html-encoding-sniffer": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz",
|
||||
"integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==",
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz",
|
||||
"integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-encoding": "^3.1.1"
|
||||
"@exodus/bytes": "^1.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/html-escaper": {
|
||||
@@ -2467,38 +2563,12 @@
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/http-proxy-agent": {
|
||||
"version": "7.0.2",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
|
||||
"integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.0",
|
||||
"debug": "^4.3.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/https": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/https/-/https-1.0.0.tgz",
|
||||
"integrity": "sha512-4EC57ddXrkaF0x83Oj8sM6SLQHAWXw90Skqu2M4AEWENZ3F02dFJE/GARA8igO79tcgYqGrD7ae4f5L3um2lgg==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/https-proxy-agent": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
|
||||
"integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "^7.1.2",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/human-id": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/human-id/-/human-id-4.1.3.tgz",
|
||||
@@ -2686,34 +2756,35 @@
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom": {
|
||||
"version": "26.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz",
|
||||
"integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==",
|
||||
"version": "29.0.2",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.0.2.tgz",
|
||||
"integrity": "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cssstyle": "^4.2.1",
|
||||
"data-urls": "^5.0.0",
|
||||
"decimal.js": "^10.5.0",
|
||||
"html-encoding-sniffer": "^4.0.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"@asamuzakjp/css-color": "^5.1.5",
|
||||
"@asamuzakjp/dom-selector": "^7.0.6",
|
||||
"@bramus/specificity": "^2.4.2",
|
||||
"@csstools/css-syntax-patches-for-csstree": "^1.1.1",
|
||||
"@exodus/bytes": "^1.15.0",
|
||||
"css-tree": "^3.2.1",
|
||||
"data-urls": "^7.0.0",
|
||||
"decimal.js": "^10.6.0",
|
||||
"html-encoding-sniffer": "^6.0.0",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"nwsapi": "^2.2.16",
|
||||
"parse5": "^7.2.1",
|
||||
"rrweb-cssom": "^0.8.0",
|
||||
"lru-cache": "^11.2.7",
|
||||
"parse5": "^8.0.0",
|
||||
"saxes": "^6.0.0",
|
||||
"symbol-tree": "^3.2.4",
|
||||
"tough-cookie": "^5.1.1",
|
||||
"tough-cookie": "^6.0.1",
|
||||
"undici": "^7.24.5",
|
||||
"w3c-xmlserializer": "^5.0.0",
|
||||
"webidl-conversions": "^7.0.0",
|
||||
"whatwg-encoding": "^3.1.1",
|
||||
"whatwg-mimetype": "^4.0.0",
|
||||
"whatwg-url": "^14.1.1",
|
||||
"ws": "^8.18.0",
|
||||
"webidl-conversions": "^8.0.1",
|
||||
"whatwg-mimetype": "^5.0.0",
|
||||
"whatwg-url": "^16.0.1",
|
||||
"xml-name-validator": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"canvas": "^3.0.0"
|
||||
@@ -2724,6 +2795,48 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom/node_modules/entities": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz",
|
||||
"integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom/node_modules/parse5": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz",
|
||||
"integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^8.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom/node_modules/undici": {
|
||||
"version": "7.25.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz",
|
||||
"integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom/node_modules/whatwg-mimetype": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz",
|
||||
"integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/jsonfile": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
|
||||
@@ -2861,10 +2974,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "10.4.3",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
|
||||
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
|
||||
"license": "ISC"
|
||||
"version": "11.3.5",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz",
|
||||
"integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/markdown-table": {
|
||||
"version": "3.0.4",
|
||||
@@ -3105,6 +3221,12 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/mdn-data": {
|
||||
"version": "2.27.1",
|
||||
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz",
|
||||
"integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==",
|
||||
"license": "CC0-1.0"
|
||||
},
|
||||
"node_modules/mensch": {
|
||||
"version": "0.3.4",
|
||||
"resolved": "https://registry.npmjs.org/mensch/-/mensch-0.3.4.tgz",
|
||||
@@ -3782,12 +3904,6 @@
|
||||
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/nwsapi": {
|
||||
"version": "2.2.23",
|
||||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.23.tgz",
|
||||
"integrity": "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/outdent": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/outdent/-/outdent-0.5.0.tgz",
|
||||
@@ -4200,6 +4316,15 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/require-from-string": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
|
||||
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/resolve-from": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz",
|
||||
@@ -4231,12 +4356,6 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/rrweb-cssom": {
|
||||
"version": "0.8.0",
|
||||
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz",
|
||||
"integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/run-parallel": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
|
||||
@@ -4402,6 +4521,15 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/spawndamnit": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/spawndamnit/-/spawndamnit-3.0.1.tgz",
|
||||
@@ -4481,21 +4609,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tldts": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz",
|
||||
"integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==",
|
||||
"version": "7.0.28",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.28.tgz",
|
||||
"integrity": "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^6.1.86"
|
||||
"tldts-core": "^7.0.28"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts-core": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz",
|
||||
"integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
|
||||
"version": "7.0.28",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.28.tgz",
|
||||
"integrity": "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/to-regex-range": {
|
||||
@@ -4512,27 +4640,27 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tough-cookie": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
|
||||
"integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz",
|
||||
"integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"tldts": "^6.1.32"
|
||||
"tldts": "^7.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
|
||||
"integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==",
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz",
|
||||
"integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"punycode": "^2.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/trough": {
|
||||
@@ -4778,12 +4906,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz",
|
||||
"integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding": {
|
||||
@@ -4809,16 +4937,17 @@
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "14.2.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
|
||||
"integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==",
|
||||
"version": "16.0.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz",
|
||||
"integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "^5.1.0",
|
||||
"webidl-conversions": "^7.0.0"
|
||||
"@exodus/bytes": "^1.11.0",
|
||||
"tr46": "^6.0.0",
|
||||
"webidl-conversions": "^8.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
@@ -4894,8 +5023,8 @@
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"chrome-launcher": "^1.2.1",
|
||||
"defuddle": "^0.14.0",
|
||||
"jsdom": "^26.0.0",
|
||||
"defuddle": "^0.17.0",
|
||||
"jsdom": "^29.0.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"turndown": "^7.2.0",
|
||||
|
||||
@@ -20,5 +20,8 @@
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pptxgenjs": "^4.0.1",
|
||||
"sharp": "^0.34.5"
|
||||
},
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,21 @@ English | [简体中文](./CHANGELOG.zh-CN.md)
|
||||
The format is inspired by Keep a Changelog, and the project follows Semantic
|
||||
Versioning.
|
||||
|
||||
## [0.1.2] - 2026-04-21
|
||||
|
||||
#### Changed
|
||||
|
||||
- Upgraded Defuddle to 0.17.0 and jsdom to 29.0.2 for generic extraction.
|
||||
- Added an `@xmldom/xmldom` override to keep the optional Defuddle MathML
|
||||
dependency chain on a non-vulnerable release.
|
||||
|
||||
#### Fixed
|
||||
|
||||
- Fixed X/Twitter video extraction for single posts and X Articles by selecting
|
||||
the highest-bitrate MP4 variant instead of the preview image URL.
|
||||
- Fixed X Article media rendering so video entities are emitted as
|
||||
`[video](...)` links instead of image embeds.
|
||||
|
||||
## [0.1.1] - 2026-03-27
|
||||
|
||||
#### Added
|
||||
|
||||
@@ -4,6 +4,18 @@
|
||||
|
||||
格式参考 Keep a Changelog,版本号遵循 Semantic Versioning。
|
||||
|
||||
## [0.1.2] - 2026-04-21
|
||||
|
||||
### 变更
|
||||
|
||||
- 将 Defuddle 升级到 0.17.0、jsdom 升级到 29.0.2,用于通用页面提取。
|
||||
- 新增 `@xmldom/xmldom` override,使 Defuddle 的可选 MathML 依赖链保持在无漏洞版本。
|
||||
|
||||
### 修复
|
||||
|
||||
- 修复 X/Twitter 单条内容和 X Article 的视频提取逻辑,改为选择最高码率 MP4 变体,而不是预览图 URL。
|
||||
- 修复 X Article 媒体渲染,视频实体现在输出为 `[video](...)` 链接,而不是图片嵌入。
|
||||
|
||||
## [0.1.1] - 2026-03-27
|
||||
|
||||
### 新增
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 0,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "baoyu-fetch",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"chrome-launcher": "^1.2.1",
|
||||
"defuddle": "^0.14.0",
|
||||
"jsdom": "^26.0.0",
|
||||
"defuddle": "^0.17.0",
|
||||
"jsdom": "^29.0.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"turndown": "^7.2.0",
|
||||
@@ -24,11 +25,22 @@
|
||||
},
|
||||
},
|
||||
},
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13",
|
||||
},
|
||||
"packages": {
|
||||
"@asamuzakjp/css-color": ["@asamuzakjp/css-color@3.2.0", "", { "dependencies": { "@csstools/css-calc": "^2.1.3", "@csstools/css-color-parser": "^3.0.9", "@csstools/css-parser-algorithms": "^3.0.4", "@csstools/css-tokenizer": "^3.0.3", "lru-cache": "^10.4.3" } }, "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw=="],
|
||||
"@asamuzakjp/css-color": ["@asamuzakjp/css-color@5.1.11", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@csstools/css-calc": "^3.2.0", "@csstools/css-color-parser": "^4.1.0", "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg=="],
|
||||
|
||||
"@asamuzakjp/dom-selector": ["@asamuzakjp/dom-selector@7.1.1", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@asamuzakjp/nwsapi": "^2.3.9", "bidi-js": "^1.0.3", "css-tree": "^3.2.1", "is-potential-custom-element-name": "^1.0.1" } }, "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ=="],
|
||||
|
||||
"@asamuzakjp/generational-cache": ["@asamuzakjp/generational-cache@1.0.1", "", {}, "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg=="],
|
||||
|
||||
"@asamuzakjp/nwsapi": ["@asamuzakjp/nwsapi@2.3.9", "", {}, "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q=="],
|
||||
|
||||
"@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="],
|
||||
|
||||
"@bramus/specificity": ["@bramus/specificity@2.4.2", "", { "dependencies": { "css-tree": "^3.0.0" }, "bin": { "specificity": "bin/cli.js" } }, "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw=="],
|
||||
|
||||
"@changesets/apply-release-plan": ["@changesets/apply-release-plan@7.1.0", "", { "dependencies": { "@changesets/config": "^3.1.3", "@changesets/get-version-range-type": "^0.4.0", "@changesets/git": "^3.0.4", "@changesets/should-skip-package": "^0.1.2", "@changesets/types": "^6.1.0", "@manypkg/get-packages": "^1.1.3", "detect-indent": "^6.0.0", "fs-extra": "^7.0.1", "lodash.startcase": "^4.4.0", "outdent": "^0.5.0", "prettier": "^2.7.1", "resolve-from": "^5.0.0", "semver": "^7.5.3" } }, "sha512-yq8ML3YS7koKQ/9bk1PqO0HMzApIFNwjlwCnwFEXMzNe8NpzeeYYKCmnhWJGkN8g7E51MnWaSbqRcTcdIxUgnQ=="],
|
||||
|
||||
"@changesets/assemble-release-plan": ["@changesets/assemble-release-plan@6.0.9", "", { "dependencies": { "@changesets/errors": "^0.2.0", "@changesets/get-dependents-graph": "^2.1.3", "@changesets/should-skip-package": "^0.1.2", "@changesets/types": "^6.1.0", "@manypkg/get-packages": "^1.1.3", "semver": "^7.5.3" } }, "sha512-tPgeeqCHIwNo8sypKlS3gOPmsS3wP0zHt67JDuL20P4QcXiw/O4Hl7oXiuLnP9yg+rXLQ2sScdV1Kkzde61iSQ=="],
|
||||
@@ -63,15 +75,19 @@
|
||||
|
||||
"@changesets/write": ["@changesets/write@0.4.0", "", { "dependencies": { "@changesets/types": "^6.1.0", "fs-extra": "^7.0.1", "human-id": "^4.1.1", "prettier": "^2.7.1" } }, "sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q=="],
|
||||
|
||||
"@csstools/color-helpers": ["@csstools/color-helpers@5.1.0", "", {}, "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA=="],
|
||||
"@csstools/color-helpers": ["@csstools/color-helpers@6.0.2", "", {}, "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q=="],
|
||||
|
||||
"@csstools/css-calc": ["@csstools/css-calc@2.1.4", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ=="],
|
||||
"@csstools/css-calc": ["@csstools/css-calc@3.2.0", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w=="],
|
||||
|
||||
"@csstools/css-color-parser": ["@csstools/css-color-parser@3.1.0", "", { "dependencies": { "@csstools/color-helpers": "^5.1.0", "@csstools/css-calc": "^2.1.4" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA=="],
|
||||
"@csstools/css-color-parser": ["@csstools/css-color-parser@4.1.0", "", { "dependencies": { "@csstools/color-helpers": "^6.0.2", "@csstools/css-calc": "^3.2.0" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ=="],
|
||||
|
||||
"@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@3.0.5", "", { "peerDependencies": { "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ=="],
|
||||
"@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@4.0.0", "", { "peerDependencies": { "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w=="],
|
||||
|
||||
"@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="],
|
||||
"@csstools/css-syntax-patches-for-csstree": ["@csstools/css-syntax-patches-for-csstree@1.1.3", "", { "peerDependencies": { "css-tree": "^3.2.1" }, "optionalPeers": ["css-tree"] }, "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg=="],
|
||||
|
||||
"@csstools/css-tokenizer": ["@csstools/css-tokenizer@4.0.0", "", {}, "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA=="],
|
||||
|
||||
"@exodus/bytes": ["@exodus/bytes@1.15.0", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" }, "optionalPeers": ["@noble/hashes"] }, "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ=="],
|
||||
|
||||
"@inquirer/external-editor": ["@inquirer/external-editor@1.0.3", "", { "dependencies": { "chardet": "^2.1.1", "iconv-lite": "^0.7.0" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA=="],
|
||||
|
||||
@@ -107,9 +123,7 @@
|
||||
|
||||
"@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="],
|
||||
|
||||
"@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="],
|
||||
|
||||
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
|
||||
"@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="],
|
||||
|
||||
"ansi-colors": ["ansi-colors@4.1.3", "", {}, "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw=="],
|
||||
|
||||
@@ -123,6 +137,8 @@
|
||||
|
||||
"better-path-resolve": ["better-path-resolve@1.0.0", "", { "dependencies": { "is-windows": "^1.0.0" } }, "sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g=="],
|
||||
|
||||
"bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="],
|
||||
@@ -143,13 +159,13 @@
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-tree": ["css-tree@3.2.1", "", { "dependencies": { "mdn-data": "2.27.1", "source-map-js": "^1.2.1" } }, "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
||||
|
||||
"cssstyle": ["cssstyle@4.6.0", "", { "dependencies": { "@asamuzakjp/css-color": "^3.2.0", "rrweb-cssom": "^0.8.0" } }, "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg=="],
|
||||
|
||||
"data-urls": ["data-urls@5.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0" } }, "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg=="],
|
||||
"data-urls": ["data-urls@7.0.0", "", { "dependencies": { "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.0" } }, "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
@@ -157,7 +173,7 @@
|
||||
|
||||
"decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="],
|
||||
|
||||
"defuddle": ["defuddle@0.14.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "linkedom": "^0.18.12", "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-btavZGd1WgiVqrVM62WGRXMUi/aU7ckTZiq0xXWLZMHvzIqNZjwIFQEDRx8MarD7fIgsB90NXZ9xHJkKtapt2Q=="],
|
||||
"defuddle": ["defuddle@0.17.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "linkedom": "^0.18.12", "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-rjRb9kxgrJLFhFJjXTSAitbmmSyvjEqGCi5JiyM6ImOf4leC/O+f7TPVPAymk/gnENFTrn9j7T/LTfW9IVwiLw=="],
|
||||
|
||||
"dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="],
|
||||
|
||||
@@ -203,16 +219,12 @@
|
||||
|
||||
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
|
||||
|
||||
"html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="],
|
||||
"html-encoding-sniffer": ["html-encoding-sniffer@6.0.0", "", { "dependencies": { "@exodus/bytes": "^1.6.0" } }, "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg=="],
|
||||
|
||||
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="],
|
||||
|
||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
||||
|
||||
"human-id": ["human-id@4.1.3", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-tsYlhAYpjCKa//8rXZ9DqKEawhPoSytweBC2eNvcaDK+57RZLHGqNs3PZTQO6yekLFSuvA6AlnAfrw1uBvtb+Q=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
|
||||
@@ -241,7 +253,7 @@
|
||||
|
||||
"js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="],
|
||||
|
||||
"jsdom": ["jsdom@26.1.0", "", { "dependencies": { "cssstyle": "^4.2.1", "data-urls": "^5.0.0", "decimal.js": "^10.5.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.16", "parse5": "^7.2.1", "rrweb-cssom": "^0.8.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^5.1.1", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.1.1", "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg=="],
|
||||
"jsdom": ["jsdom@29.0.2", "", { "dependencies": { "@asamuzakjp/css-color": "^5.1.5", "@asamuzakjp/dom-selector": "^7.0.6", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.1", "@exodus/bytes": "^1.15.0", "css-tree": "^3.2.1", "data-urls": "^7.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "is-potential-custom-element-name": "^1.0.1", "lru-cache": "^11.2.7", "parse5": "^8.0.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.1", "undici": "^7.24.5", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.1", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w=="],
|
||||
|
||||
"jsonfile": ["jsonfile@4.0.0", "", { "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg=="],
|
||||
|
||||
@@ -255,7 +267,7 @@
|
||||
|
||||
"longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="],
|
||||
|
||||
"lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
||||
"lru-cache": ["lru-cache@11.3.5", "", {}, "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw=="],
|
||||
|
||||
"markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="],
|
||||
|
||||
@@ -285,6 +297,8 @@
|
||||
|
||||
"mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],
|
||||
|
||||
"mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="],
|
||||
|
||||
"merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="],
|
||||
|
||||
"micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="],
|
||||
@@ -351,8 +365,6 @@
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"nwsapi": ["nwsapi@2.2.23", "", {}, "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ=="],
|
||||
|
||||
"outdent": ["outdent@0.5.0", "", {}, "sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q=="],
|
||||
|
||||
"p-filter": ["p-filter@2.1.0", "", { "dependencies": { "p-map": "^2.0.0" } }, "sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw=="],
|
||||
@@ -397,12 +409,12 @@
|
||||
|
||||
"remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="],
|
||||
|
||||
"require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
|
||||
|
||||
"resolve-from": ["resolve-from@5.0.0", "", {}, "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw=="],
|
||||
|
||||
"reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="],
|
||||
|
||||
"rrweb-cssom": ["rrweb-cssom@0.8.0", "", {}, "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw=="],
|
||||
|
||||
"run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
@@ -419,6 +431,8 @@
|
||||
|
||||
"slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="],
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
|
||||
"spawndamnit": ["spawndamnit@3.0.1", "", { "dependencies": { "cross-spawn": "^7.0.5", "signal-exit": "^4.0.1" } }, "sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg=="],
|
||||
|
||||
"sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="],
|
||||
@@ -433,15 +447,15 @@
|
||||
|
||||
"term-size": ["term-size@2.2.1", "", {}, "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg=="],
|
||||
|
||||
"tldts": ["tldts@6.1.86", "", { "dependencies": { "tldts-core": "^6.1.86" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ=="],
|
||||
"tldts": ["tldts@7.0.28", "", { "dependencies": { "tldts-core": "^7.0.28" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw=="],
|
||||
|
||||
"tldts-core": ["tldts-core@6.1.86", "", {}, "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA=="],
|
||||
"tldts-core": ["tldts-core@7.0.28", "", {}, "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ=="],
|
||||
|
||||
"to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
|
||||
|
||||
"tough-cookie": ["tough-cookie@5.1.2", "", { "dependencies": { "tldts": "^6.1.32" } }, "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A=="],
|
||||
"tough-cookie": ["tough-cookie@6.0.1", "", { "dependencies": { "tldts": "^7.0.5" } }, "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw=="],
|
||||
|
||||
"tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="],
|
||||
"tr46": ["tr46@6.0.0", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw=="],
|
||||
|
||||
"trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="],
|
||||
|
||||
@@ -453,6 +467,8 @@
|
||||
|
||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||
|
||||
"undici": ["undici@7.25.0", "", {}, "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ=="],
|
||||
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="],
|
||||
@@ -473,13 +489,11 @@
|
||||
|
||||
"w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="],
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="],
|
||||
"webidl-conversions": ["webidl-conversions@8.0.1", "", {}, "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
"whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
|
||||
"whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "^1.11.0", "tr46": "^6.0.0", "webidl-conversions": "^8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="],
|
||||
|
||||
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
||||
|
||||
@@ -503,11 +517,13 @@
|
||||
|
||||
"htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||
|
||||
"jsdom/parse5": ["parse5@8.0.1", "", { "dependencies": { "entities": "^8.0.0" } }, "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw=="],
|
||||
|
||||
"mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="],
|
||||
|
||||
"read-yaml-file/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
|
||||
|
||||
"whatwg-encoding/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
"jsdom/parse5/entities": ["entities@8.0.0", "", {}, "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA=="],
|
||||
|
||||
"read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
|
||||
}
|
||||
|
||||
@@ -46,8 +46,8 @@
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"chrome-launcher": "^1.2.1",
|
||||
"defuddle": "^0.14.0",
|
||||
"jsdom": "^26.0.0",
|
||||
"defuddle": "^0.17.0",
|
||||
"jsdom": "^29.0.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"turndown": "^7.2.0",
|
||||
@@ -61,5 +61,8 @@
|
||||
"@types/jsdom": "^21.1.7",
|
||||
"@types/ws": "^8.18.1",
|
||||
"typescript": "^5.9.2"
|
||||
},
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -339,4 +339,105 @@ describe("x article extraction", () => {
|
||||
expect(content.markdown).toContain("https://example.com/report");
|
||||
expect(content.markdown).not.toContain("https://t.co/example");
|
||||
});
|
||||
|
||||
test("renders article video media as the highest bitrate mp4 link", () => {
|
||||
const payload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2046628728210350366",
|
||||
legacy: {
|
||||
full_text: "Fallback text",
|
||||
favorite_count: 12,
|
||||
retweet_count: 3,
|
||||
reply_count: 1,
|
||||
created_at: "Tue Apr 21 16:34:47 +0000 2026",
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Google AI Studio",
|
||||
screen_name: "GoogleAIStudio",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: "Article with video",
|
||||
media_entities: [
|
||||
{
|
||||
media_id: "2046627051822530560",
|
||||
media_info: {
|
||||
__typename: "ApiVideo",
|
||||
variants: [
|
||||
{
|
||||
bit_rate: 2176000,
|
||||
content_type: "video/mp4",
|
||||
url: "https://video.twimg.com/amplify_video/2046627051822530560/vid/avc1/1280x720/medium.mp4",
|
||||
},
|
||||
{
|
||||
content_type: "application/x-mpegURL",
|
||||
url: "https://video.twimg.com/amplify_video/2046627051822530560/pl/playlist.m3u8",
|
||||
},
|
||||
{
|
||||
bit_rate: 10368000,
|
||||
content_type: "video/mp4",
|
||||
url: "https://video.twimg.com/amplify_video/2046627051822530560/vid/avc1/1920x1080/high.mp4",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
content_state: {
|
||||
blocks: [
|
||||
{
|
||||
type: "atomic",
|
||||
text: " ",
|
||||
data: {},
|
||||
entityRanges: [{ key: 0, length: 1, offset: 0 }],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
],
|
||||
entityMap: [
|
||||
{
|
||||
key: "0",
|
||||
value: {
|
||||
type: "MEDIA",
|
||||
mutability: "Immutable",
|
||||
data: {
|
||||
mediaItems: [{ mediaId: "2046627051822530560" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractArticleDocumentFromPayload(
|
||||
payload,
|
||||
"2046628728210350366",
|
||||
"https://x.com/GoogleAIStudio/status/2046628728210350366",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
|
||||
const content = document?.content[0];
|
||||
expect(content?.type).toBe("markdown");
|
||||
if (!content || content.type !== "markdown") {
|
||||
throw new Error("Expected markdown content");
|
||||
}
|
||||
|
||||
expect(content.markdown).toBe(
|
||||
"[video](https://video.twimg.com/amplify_video/2046627051822530560/vid/avc1/1920x1080/high.mp4)",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -184,4 +184,77 @@ describe("x single tweet extraction", () => {
|
||||
"Quoted Author (@quoted_author)\n\nQuoted tweet text\n\nphoto: https://pbs.twimg.com/media/quoted?format=jpg&name=4096x4096",
|
||||
});
|
||||
});
|
||||
|
||||
test("uses the highest bitrate mp4 variant for tweet video media", () => {
|
||||
const payload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2046628728210350366",
|
||||
legacy: {
|
||||
full_text: "Video post https://t.co/video",
|
||||
favorite_count: 12,
|
||||
retweet_count: 3,
|
||||
reply_count: 1,
|
||||
created_at: "Tue Apr 21 16:34:47 +0000 2026",
|
||||
extended_entities: {
|
||||
media: [
|
||||
{
|
||||
type: "video",
|
||||
media_url_https: "https://pbs.twimg.com/amplify_video_thumb/2046627051822530560/img/poster.jpg",
|
||||
url: "https://t.co/video",
|
||||
video_info: {
|
||||
variants: [
|
||||
{
|
||||
content_type: "application/x-mpegURL",
|
||||
url: "https://video.twimg.com/amplify_video/2046627051822530560/pl/playlist.m3u8",
|
||||
},
|
||||
{
|
||||
bitrate: 256000,
|
||||
content_type: "video/mp4",
|
||||
url: "https://video.twimg.com/amplify_video/2046627051822530560/vid/avc1/480x270/low.mp4",
|
||||
},
|
||||
{
|
||||
bitrate: 10368000,
|
||||
content_type: "video/mp4",
|
||||
url: "https://video.twimg.com/amplify_video/2046627051822530560/vid/avc1/1920x1080/high.mp4",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Google AI Studio",
|
||||
screen_name: "GoogleAIStudio",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractSingleTweetDocumentFromPayload(
|
||||
payload,
|
||||
"2046628728210350366",
|
||||
"https://x.com/GoogleAIStudio/status/2046628728210350366",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
|
||||
const listBlock = document?.content.find((block) => block.type === "list");
|
||||
expect(listBlock).toEqual({
|
||||
type: "list",
|
||||
ordered: false,
|
||||
items: [
|
||||
"video: https://video.twimg.com/amplify_video/2046627051822530560/vid/avc1/1920x1080/high.mp4",
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,18 +9,43 @@ import {
|
||||
getUser,
|
||||
isRecord,
|
||||
normalizeTitle,
|
||||
resolveBestXVideoVariantUrl,
|
||||
toHighResXImageUrl,
|
||||
toXTweet,
|
||||
} from "./shared";
|
||||
import type { JsonObject } from "./types";
|
||||
|
||||
function resolveArticleMediaUrl(mediaInfo: JsonObject): string {
|
||||
interface ArticleMedia {
|
||||
kind: "image" | "video";
|
||||
url: string;
|
||||
}
|
||||
|
||||
function resolveArticleMedia(mediaInfo: JsonObject): ArticleMedia | null {
|
||||
const videoUrl = resolveBestXVideoVariantUrl(mediaInfo);
|
||||
if (videoUrl) {
|
||||
return {
|
||||
kind: "video",
|
||||
url: videoUrl,
|
||||
};
|
||||
}
|
||||
|
||||
const rawUrl =
|
||||
(typeof mediaInfo.original_img_url === "string" && mediaInfo.original_img_url) ||
|
||||
(typeof mediaInfo.url === "string" && mediaInfo.url) ||
|
||||
"";
|
||||
|
||||
return rawUrl ? toHighResXImageUrl(rawUrl) : "";
|
||||
if (!rawUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "image",
|
||||
url: toHighResXImageUrl(rawUrl),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveArticleMediaUrl(mediaInfo: JsonObject): string {
|
||||
return resolveArticleMedia(mediaInfo)?.url ?? "";
|
||||
}
|
||||
|
||||
function normalizeEntityMap(entityMap: unknown): Map<string, JsonObject> {
|
||||
@@ -139,8 +164,8 @@ function getTweetId(entityMap: Map<string, JsonObject>, entityKey: unknown): str
|
||||
return data.tweetId;
|
||||
}
|
||||
|
||||
function buildMediaUrlMap(articleResult: JsonObject): Map<string, string> {
|
||||
const mediaMap = new Map<string, string>();
|
||||
function buildMediaMap(articleResult: JsonObject): Map<string, ArticleMedia> {
|
||||
const mediaMap = new Map<string, ArticleMedia>();
|
||||
const mediaEntities = Array.isArray(articleResult.media_entities) ? articleResult.media_entities : [];
|
||||
|
||||
for (const entity of mediaEntities) {
|
||||
@@ -148,25 +173,28 @@ function buildMediaUrlMap(articleResult: JsonObject): Map<string, string> {
|
||||
continue;
|
||||
}
|
||||
|
||||
const mediaInfo = entity.media_info;
|
||||
const url = resolveArticleMediaUrl(mediaInfo);
|
||||
if (url) {
|
||||
mediaMap.set(entity.media_id, url);
|
||||
const media = resolveArticleMedia(entity.media_info);
|
||||
if (media) {
|
||||
mediaMap.set(entity.media_id, media);
|
||||
}
|
||||
}
|
||||
|
||||
const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null;
|
||||
if (coverMedia && typeof coverMedia.media_id === "string" && isRecord(coverMedia.media_info)) {
|
||||
const url = resolveArticleMediaUrl(coverMedia.media_info);
|
||||
if (url) {
|
||||
mediaMap.set(coverMedia.media_id, url);
|
||||
const media = resolveArticleMedia(coverMedia.media_info);
|
||||
if (media) {
|
||||
mediaMap.set(coverMedia.media_id, media);
|
||||
}
|
||||
}
|
||||
|
||||
return mediaMap;
|
||||
}
|
||||
|
||||
function getMediaMarkdown(entityMap: Map<string, JsonObject>, entityKey: unknown, mediaMap: Map<string, string>): string[] {
|
||||
function getMediaMarkdown(
|
||||
entityMap: Map<string, JsonObject>,
|
||||
entityKey: unknown,
|
||||
mediaMap: Map<string, ArticleMedia>,
|
||||
): string[] {
|
||||
const key =
|
||||
typeof entityKey === "string" || typeof entityKey === "number"
|
||||
? String(entityKey)
|
||||
@@ -182,19 +210,19 @@ function getMediaMarkdown(entityMap: Map<string, JsonObject>, entityKey: unknown
|
||||
|
||||
const data = isRecord(entity.data) ? entity.data : {};
|
||||
const mediaItems = Array.isArray(data.mediaItems) ? data.mediaItems : [];
|
||||
const urls: string[] = [];
|
||||
const media: ArticleMedia[] = [];
|
||||
|
||||
for (const item of mediaItems) {
|
||||
if (!isRecord(item) || typeof item.mediaId !== "string") {
|
||||
continue;
|
||||
}
|
||||
const url = mediaMap.get(item.mediaId);
|
||||
if (url && !urls.includes(url)) {
|
||||
urls.push(url);
|
||||
const mediaItem = mediaMap.get(item.mediaId);
|
||||
if (mediaItem && !media.some((value) => value.url === mediaItem.url)) {
|
||||
media.push(mediaItem);
|
||||
}
|
||||
}
|
||||
|
||||
return urls.map((url) => ``);
|
||||
return media.map((item) => item.kind === "image" ? `` : `[video](${item.url})`);
|
||||
}
|
||||
|
||||
function resolveTweetMarkdown(payloads: unknown[], tweetId: string, pageUrl: string): string | null {
|
||||
@@ -250,7 +278,7 @@ function replaceLinkEntities(text: string, block: JsonObject, entityMap: Map<str
|
||||
function renderAtomicBlock(
|
||||
block: JsonObject,
|
||||
entityMap: Map<string, JsonObject>,
|
||||
mediaMap: Map<string, string>,
|
||||
mediaMap: Map<string, ArticleMedia>,
|
||||
payloads: unknown[],
|
||||
pageUrl: string,
|
||||
): string | null {
|
||||
@@ -293,7 +321,7 @@ function renderAtomicBlock(
|
||||
function renderArticleBlocks(
|
||||
blocks: unknown[],
|
||||
entityMap: Map<string, JsonObject>,
|
||||
mediaMap: Map<string, string>,
|
||||
mediaMap: Map<string, ArticleMedia>,
|
||||
payloads: unknown[],
|
||||
pageUrl: string,
|
||||
): string {
|
||||
@@ -396,7 +424,7 @@ export function extractArticleDocumentFromPayload(
|
||||
const contentState = isRecord(articleResult.content_state) ? articleResult.content_state : {};
|
||||
const blocks = Array.isArray(contentState.blocks) ? contentState.blocks : [];
|
||||
const entityMap = normalizeEntityMap(contentState.entityMap);
|
||||
const mediaMap = buildMediaUrlMap(articleResult);
|
||||
const mediaMap = buildMediaMap(articleResult);
|
||||
const richMarkdown = renderArticleBlocks(blocks, entityMap, mediaMap, payloads, pageUrl);
|
||||
const plainText = typeof articleResult.plain_text === "string" ? articleResult.plain_text.trim() : "";
|
||||
const markdown = richMarkdown || plainText || getTweetText(tweet);
|
||||
|
||||
@@ -251,6 +251,39 @@ export function toHighResXImageUrl(rawUrl: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
function getVideoVariantBitrate(variant: JsonObject): number {
|
||||
const value = variant.bitrate ?? variant.bit_rate;
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function getVideoVariantContentType(variant: JsonObject): string {
|
||||
const value = variant.content_type ?? variant.contentType;
|
||||
return typeof value === "string" ? value.toLowerCase() : "";
|
||||
}
|
||||
|
||||
export function resolveBestXVideoVariantUrl(mediaInfo: unknown): string | undefined {
|
||||
if (!isRecord(mediaInfo)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const variantsSource =
|
||||
Array.isArray(mediaInfo.variants)
|
||||
? mediaInfo.variants
|
||||
: isRecord(mediaInfo.video_info) && Array.isArray(mediaInfo.video_info.variants)
|
||||
? mediaInfo.video_info.variants
|
||||
: [];
|
||||
|
||||
const variants = variantsSource
|
||||
.filter(
|
||||
(variant): variant is JsonObject =>
|
||||
isRecord(variant) && typeof variant.url === "string" && variant.url.length > 0,
|
||||
)
|
||||
.filter((variant) => getVideoVariantContentType(variant) === "video/mp4")
|
||||
.sort((left, right) => getVideoVariantBitrate(right) - getVideoVariantBitrate(left));
|
||||
|
||||
return typeof variants[0]?.url === "string" ? variants[0].url : undefined;
|
||||
}
|
||||
|
||||
export function getTweetMedia(tweet: JsonObject): XMedia[] {
|
||||
const legacy = getLegacy(tweet);
|
||||
const extendedEntities = isRecord(legacy.extended_entities) ? legacy.extended_entities : emptyObject();
|
||||
@@ -268,10 +301,14 @@ export function getTweetMedia(tweet: JsonObject): XMedia[] {
|
||||
alt: typeof value.ext_alt_text === "string" ? value.ext_alt_text : undefined,
|
||||
};
|
||||
}
|
||||
if ((value.type === "video" || value.type === "animated_gif") && typeof value.media_url_https === "string") {
|
||||
if (value.type === "video" || value.type === "animated_gif") {
|
||||
const videoUrl = resolveBestXVideoVariantUrl(value);
|
||||
if (!videoUrl) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: value.type,
|
||||
url: value.media_url_https,
|
||||
url: videoUrl,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -10,6 +10,20 @@ const PACKAGE_DEPENDENCY_SECTIONS = [
|
||||
|
||||
const SKIPPED_DIRS = new Set([".git", ".clawhub", ".clawdhub", "node_modules", "out", "dist", "build"]);
|
||||
const SKIPPED_FILES = new Set([".DS_Store", "bun.lockb"]);
|
||||
const MIME_MAP = {
|
||||
".md": "text/markdown",
|
||||
".ts": "text/plain",
|
||||
".js": "text/javascript",
|
||||
".mjs": "text/javascript",
|
||||
".json": "application/json",
|
||||
".yml": "text/yaml",
|
||||
".yaml": "text/yaml",
|
||||
".txt": "text/plain",
|
||||
".html": "text/html",
|
||||
".css": "text/css",
|
||||
".xml": "text/xml",
|
||||
".svg": "image/svg+xml",
|
||||
};
|
||||
|
||||
export async function listReleaseFiles(root) {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
@@ -40,9 +54,10 @@ export async function listReleaseFiles(root) {
|
||||
}
|
||||
|
||||
export async function validateSelfContainedRelease(root) {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const files = await listReleaseFiles(root);
|
||||
for (const file of files.filter((entry) => path.posix.basename(entry.relPath) === "package.json")) {
|
||||
const packageDir = path.resolve(root, fromPosixRel(path.posix.dirname(file.relPath)));
|
||||
const packageDir = path.resolve(resolvedRoot, fromPosixRel(path.posix.dirname(file.relPath)));
|
||||
const packageJson = JSON.parse(file.bytes.toString("utf8"));
|
||||
for (const section of PACKAGE_DEPENDENCY_SECTIONS) {
|
||||
const dependencies = packageJson[section];
|
||||
@@ -51,7 +66,7 @@ export async function validateSelfContainedRelease(root) {
|
||||
for (const [name, spec] of Object.entries(dependencies)) {
|
||||
if (typeof spec !== "string" || !spec.startsWith("file:")) continue;
|
||||
const targetDir = path.resolve(packageDir, spec.slice(5));
|
||||
if (!isWithinRoot(root, targetDir)) {
|
||||
if (!isWithinRoot(resolvedRoot, targetDir)) {
|
||||
throw new Error(
|
||||
`Release target is not self-contained: ${file.relPath} depends on ${name} via ${spec}`,
|
||||
);
|
||||
@@ -61,9 +76,35 @@ export async function validateSelfContainedRelease(root) {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const target of getPackageBinTargets(packageJson)) {
|
||||
const targetPath = path.resolve(packageDir, target);
|
||||
if (!isWithinRoot(resolvedRoot, targetPath)) {
|
||||
throw new Error(`Release target is not self-contained: ${file.relPath} bin points to ${target}`);
|
||||
}
|
||||
await fs.access(targetPath).catch(() => {
|
||||
throw new Error(`Missing package bin target for release: ${file.relPath} -> ${target}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function mimeType(relPath) {
|
||||
const ext = path.extname(relPath).toLowerCase();
|
||||
return MIME_MAP[ext] || "text/plain";
|
||||
}
|
||||
|
||||
function getPackageBinTargets(packageJson) {
|
||||
const bin = packageJson.bin;
|
||||
if (typeof bin === "string" && bin.trim()) {
|
||||
return [bin.trim()];
|
||||
}
|
||||
if (!bin || typeof bin !== "object" || Array.isArray(bin)) {
|
||||
return [];
|
||||
}
|
||||
return Object.values(bin).filter((value) => typeof value === "string" && value.trim());
|
||||
}
|
||||
|
||||
function fromPosixRel(relPath) {
|
||||
return relPath === "." ? "." : relPath.split("/").join(path.sep);
|
||||
}
|
||||
|
||||
@@ -65,6 +65,40 @@ test("validateSelfContainedRelease accepts file dependencies that stay within th
|
||||
await assert.doesNotReject(() => validateSelfContainedRelease(root));
|
||||
});
|
||||
|
||||
test("validateSelfContainedRelease accepts package bin targets inside the release root", async (t) => {
|
||||
const root = await makeTempDir("baoyu-release-bin-ok-");
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
|
||||
await writeJson(path.join(root, "scripts", "package.json"), {
|
||||
name: "test-skill-scripts",
|
||||
version: "1.0.0",
|
||||
bin: {
|
||||
"test-skill": "./test-skill",
|
||||
},
|
||||
});
|
||||
await writeFile(path.join(root, "scripts", "test-skill"), "#!/usr/bin/env sh\n");
|
||||
|
||||
await assert.doesNotReject(() => validateSelfContainedRelease(root));
|
||||
});
|
||||
|
||||
test("validateSelfContainedRelease rejects missing package bin targets", async (t) => {
|
||||
const root = await makeTempDir("baoyu-release-bin-missing-");
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
|
||||
await writeJson(path.join(root, "scripts", "package.json"), {
|
||||
name: "test-skill-scripts",
|
||||
version: "1.0.0",
|
||||
bin: {
|
||||
"test-skill": "./missing",
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => validateSelfContainedRelease(root),
|
||||
/Missing package bin target for release/,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateSelfContainedRelease rejects missing local file dependencies", async (t) => {
|
||||
const root = await makeTempDir("baoyu-release-missing-");
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
|
||||
@@ -5,7 +5,7 @@ import { existsSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { listReleaseFiles, validateSelfContainedRelease } from "./lib/release-files.mjs";
|
||||
import { listReleaseFiles, mimeType, validateSelfContainedRelease } from "./lib/release-files.mjs";
|
||||
|
||||
const DEFAULT_REGISTRY = "https://clawhub.ai";
|
||||
|
||||
@@ -281,26 +281,6 @@ function titleCase(value) {
|
||||
.replace(/\b\w/g, (char) => char.toUpperCase());
|
||||
}
|
||||
|
||||
const MIME_MAP = {
|
||||
".md": "text/markdown",
|
||||
".ts": "text/plain",
|
||||
".js": "text/javascript",
|
||||
".mjs": "text/javascript",
|
||||
".json": "application/json",
|
||||
".yml": "text/yaml",
|
||||
".yaml": "text/yaml",
|
||||
".txt": "text/plain",
|
||||
".html": "text/html",
|
||||
".css": "text/css",
|
||||
".xml": "text/xml",
|
||||
".svg": "image/svg+xml",
|
||||
};
|
||||
|
||||
function mimeType(relPath) {
|
||||
const ext = path.extname(relPath).toLowerCase();
|
||||
return MIME_MAP[ext] || "text/plain";
|
||||
}
|
||||
|
||||
function parseBoolean(value) {
|
||||
return ["1", "true", "yes", "on"].includes(String(value).trim().toLowerCase());
|
||||
}
|
||||
|
||||
@@ -6,47 +6,9 @@ import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
|
||||
import { listReleaseFiles, mimeType, validateSelfContainedRelease } from "./lib/release-files.mjs";
|
||||
|
||||
const DEFAULT_REGISTRY = "https://clawhub.ai";
|
||||
const TEXT_EXTENSIONS = new Set([
|
||||
"md",
|
||||
"mdx",
|
||||
"txt",
|
||||
"json",
|
||||
"json5",
|
||||
"yaml",
|
||||
"yml",
|
||||
"toml",
|
||||
"js",
|
||||
"cjs",
|
||||
"mjs",
|
||||
"ts",
|
||||
"tsx",
|
||||
"jsx",
|
||||
"py",
|
||||
"sh",
|
||||
"rb",
|
||||
"go",
|
||||
"rs",
|
||||
"swift",
|
||||
"kt",
|
||||
"java",
|
||||
"cs",
|
||||
"cpp",
|
||||
"c",
|
||||
"h",
|
||||
"hpp",
|
||||
"sql",
|
||||
"csv",
|
||||
"ini",
|
||||
"cfg",
|
||||
"env",
|
||||
"xml",
|
||||
"html",
|
||||
"css",
|
||||
"scss",
|
||||
"sass",
|
||||
"svg",
|
||||
]);
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
@@ -75,7 +37,7 @@ async function main() {
|
||||
console.log(`Roots with skills: ${roots.join(", ")}`);
|
||||
|
||||
const locals = await mapWithConcurrency(skills, options.concurrency, async (skill) => {
|
||||
const files = await listTextFiles(skill.folder);
|
||||
const files = await collectReleaseFiles(skill.folder);
|
||||
const fingerprint = buildFingerprint(files);
|
||||
return {
|
||||
...skill,
|
||||
@@ -162,7 +124,7 @@ async function main() {
|
||||
|
||||
console.log(`Publishing ${candidate.slug}@${version}`);
|
||||
try {
|
||||
const files = await listTextFiles(candidate.folder);
|
||||
const files = await collectReleaseFiles(candidate.folder);
|
||||
await publishSkill({
|
||||
registry,
|
||||
token: config.token,
|
||||
@@ -363,35 +325,9 @@ async function hasSkillMarker(folder) {
|
||||
);
|
||||
}
|
||||
|
||||
async function listTextFiles(root) {
|
||||
const files = [];
|
||||
|
||||
async function walk(folder) {
|
||||
const entries = await fs.readdir(folder, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith(".")) continue;
|
||||
if (entry.name === "node_modules") continue;
|
||||
if (entry.name === ".clawhub" || entry.name === ".clawdhub") continue;
|
||||
|
||||
const fullPath = path.join(folder, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await walk(fullPath);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) continue;
|
||||
|
||||
const relPath = path.relative(root, fullPath).split(path.sep).join("/");
|
||||
const ext = relPath.split(".").pop()?.toLowerCase() ?? "";
|
||||
if (!TEXT_EXTENSIONS.has(ext)) continue;
|
||||
|
||||
const bytes = await fs.readFile(fullPath);
|
||||
files.push({ relPath, bytes });
|
||||
}
|
||||
}
|
||||
|
||||
await walk(root);
|
||||
files.sort((left, right) => left.relPath.localeCompare(right.relPath));
|
||||
return files;
|
||||
async function collectReleaseFiles(root) {
|
||||
await validateSelfContainedRelease(root);
|
||||
return listReleaseFiles(root);
|
||||
}
|
||||
|
||||
function buildFingerprint(files) {
|
||||
@@ -421,7 +357,7 @@ async function publishSkill({ registry, token, skill, files, version, changelog,
|
||||
);
|
||||
|
||||
for (const file of files) {
|
||||
form.append("files", new Blob([file.bytes], { type: "text/plain" }), file.relPath);
|
||||
form.append("files", new Blob([file.bytes], { type: mimeType(file.relPath) }), file.relPath);
|
||||
}
|
||||
|
||||
const response = await fetch(`${registry}/api/v1/skills`, {
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
---
|
||||
name: baoyu-url-to-markdown
|
||||
description: Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.
|
||||
version: 1.60.0
|
||||
version: 1.61.0
|
||||
metadata:
|
||||
openclaw:
|
||||
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-url-to-markdown
|
||||
requires:
|
||||
anyBins:
|
||||
- bun
|
||||
- npx
|
||||
---
|
||||
|
||||
# URL to Markdown
|
||||
@@ -27,13 +26,13 @@ Concrete `AskUserQuestion` references below are examples — substitute the loca
|
||||
|
||||
## CLI Setup
|
||||
|
||||
**Important**: The CLI is provided by the npm package dependency `baoyu-fetch`.
|
||||
**Important**: The CLI source is vendored in `{baseDir}/scripts/lib`. `scripts/package.json` installs only third-party runtime dependencies.
|
||||
|
||||
**Agent Execution Instructions**:
|
||||
1. Determine this SKILL.md file's directory path as `{baseDir}`
|
||||
2. Resolve `${BUN}` runtime: if `bun` installed → `bun`; else suggest installing Bun
|
||||
3. If `{baseDir}/scripts/node_modules/.bin/baoyu-fetch` does not exist, run `${BUN} install --cwd {baseDir}/scripts`
|
||||
4. `${READER}` = `{baseDir}/scripts/node_modules/.bin/baoyu-fetch`
|
||||
3. If `{baseDir}/scripts/node_modules` does not exist, run `${BUN} install --cwd {baseDir}/scripts`
|
||||
4. `${READER}` = `{baseDir}/scripts/baoyu-fetch`
|
||||
5. Replace all `${READER}` in this document with the resolved value
|
||||
|
||||
## Preferences (EXTEND.md)
|
||||
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
exec bun "$script_dir/lib/cli.ts" "$@"
|
||||
@@ -5,22 +5,46 @@
|
||||
"": {
|
||||
"name": "baoyu-url-to-markdown-scripts",
|
||||
"dependencies": {
|
||||
"baoyu-fetch": "^0.1.2",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"chrome-launcher": "^1.2.1",
|
||||
"defuddle": "^0.17.0",
|
||||
"jsdom": "^29.0.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"turndown": "^7.2.0",
|
||||
"turndown-plugin-gfm": "^1.0.2",
|
||||
"unified": "^11.0.5",
|
||||
"ws": "^8.18.3",
|
||||
},
|
||||
},
|
||||
},
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13",
|
||||
},
|
||||
"packages": {
|
||||
"@asamuzakjp/css-color": ["@asamuzakjp/css-color@3.2.0", "", { "dependencies": { "@csstools/css-calc": "^2.1.3", "@csstools/css-color-parser": "^3.0.9", "@csstools/css-parser-algorithms": "^3.0.4", "@csstools/css-tokenizer": "^3.0.3", "lru-cache": "^10.4.3" } }, "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw=="],
|
||||
"@asamuzakjp/css-color": ["@asamuzakjp/css-color@5.1.11", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@csstools/css-calc": "^3.2.0", "@csstools/css-color-parser": "^4.1.0", "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg=="],
|
||||
|
||||
"@csstools/color-helpers": ["@csstools/color-helpers@5.1.0", "", {}, "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA=="],
|
||||
"@asamuzakjp/dom-selector": ["@asamuzakjp/dom-selector@7.1.1", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@asamuzakjp/nwsapi": "^2.3.9", "bidi-js": "^1.0.3", "css-tree": "^3.2.1", "is-potential-custom-element-name": "^1.0.1" } }, "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ=="],
|
||||
|
||||
"@csstools/css-calc": ["@csstools/css-calc@2.1.4", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ=="],
|
||||
"@asamuzakjp/generational-cache": ["@asamuzakjp/generational-cache@1.0.1", "", {}, "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg=="],
|
||||
|
||||
"@csstools/css-color-parser": ["@csstools/css-color-parser@3.1.0", "", { "dependencies": { "@csstools/color-helpers": "^5.1.0", "@csstools/css-calc": "^2.1.4" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA=="],
|
||||
"@asamuzakjp/nwsapi": ["@asamuzakjp/nwsapi@2.3.9", "", {}, "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q=="],
|
||||
|
||||
"@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@3.0.5", "", { "peerDependencies": { "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ=="],
|
||||
"@bramus/specificity": ["@bramus/specificity@2.4.2", "", { "dependencies": { "css-tree": "^3.0.0" }, "bin": { "specificity": "bin/cli.js" } }, "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw=="],
|
||||
|
||||
"@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="],
|
||||
"@csstools/color-helpers": ["@csstools/color-helpers@6.0.2", "", {}, "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q=="],
|
||||
|
||||
"@csstools/css-calc": ["@csstools/css-calc@3.2.0", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w=="],
|
||||
|
||||
"@csstools/css-color-parser": ["@csstools/css-color-parser@4.1.0", "", { "dependencies": { "@csstools/color-helpers": "^6.0.2", "@csstools/css-calc": "^3.2.0" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ=="],
|
||||
|
||||
"@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@4.0.0", "", { "peerDependencies": { "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w=="],
|
||||
|
||||
"@csstools/css-syntax-patches-for-csstree": ["@csstools/css-syntax-patches-for-csstree@1.1.3", "", { "peerDependencies": { "css-tree": "^3.2.1" }, "optionalPeers": ["css-tree"] }, "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg=="],
|
||||
|
||||
"@csstools/css-tokenizer": ["@csstools/css-tokenizer@4.0.0", "", {}, "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA=="],
|
||||
|
||||
"@exodus/bytes": ["@exodus/bytes@1.15.0", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" }, "optionalPeers": ["@noble/hashes"] }, "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ=="],
|
||||
|
||||
"@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="],
|
||||
|
||||
@@ -38,11 +62,9 @@
|
||||
|
||||
"@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="],
|
||||
|
||||
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
|
||||
|
||||
"bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="],
|
||||
|
||||
"baoyu-fetch": ["baoyu-fetch@0.1.2", "", { "dependencies": { "@mozilla/readability": "^0.6.0", "chrome-launcher": "^1.2.1", "defuddle": "^0.14.0", "jsdom": "^26.0.0", "remark-gfm": "^4.0.1", "remark-parse": "^11.0.0", "turndown": "^7.2.0", "turndown-plugin-gfm": "^1.0.2", "unified": "^11.0.5", "ws": "^8.18.3" }, "bin": { "baoyu-fetch": "dist/cli.js" } }, "sha512-VB4CEtIcoiJo3m80RrwBuYOdC257VI6kptoI1kiRcZV78VWEicWdznRoByrjddgyHTq0WWEK5i/7L67mHcVPvQ=="],
|
||||
"bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
@@ -56,13 +78,13 @@
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-tree": ["css-tree@3.2.1", "", { "dependencies": { "mdn-data": "2.27.1", "source-map-js": "^1.2.1" } }, "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
||||
|
||||
"cssstyle": ["cssstyle@4.6.0", "", { "dependencies": { "@asamuzakjp/css-color": "^3.2.0", "rrweb-cssom": "^0.8.0" } }, "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg=="],
|
||||
|
||||
"data-urls": ["data-urls@5.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0" } }, "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg=="],
|
||||
"data-urls": ["data-urls@7.0.0", "", { "dependencies": { "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.0" } }, "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
@@ -70,7 +92,7 @@
|
||||
|
||||
"decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="],
|
||||
|
||||
"defuddle": ["defuddle@0.14.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "linkedom": "^0.18.12", "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-btavZGd1WgiVqrVM62WGRXMUi/aU7ckTZiq0xXWLZMHvzIqNZjwIFQEDRx8MarD7fIgsB90NXZ9xHJkKtapt2Q=="],
|
||||
"defuddle": ["defuddle@0.17.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "linkedom": "^0.18.12", "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-rjRb9kxgrJLFhFJjXTSAitbmmSyvjEqGCi5JiyM6ImOf4leC/O+f7TPVPAymk/gnENFTrn9j7T/LTfW9IVwiLw=="],
|
||||
|
||||
"dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="],
|
||||
|
||||
@@ -84,24 +106,18 @@
|
||||
|
||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
"entities": ["entities@8.0.0", "", {}, "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA=="],
|
||||
|
||||
"escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="],
|
||||
|
||||
"extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
|
||||
|
||||
"html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="],
|
||||
"html-encoding-sniffer": ["html-encoding-sniffer@6.0.0", "", { "dependencies": { "@exodus/bytes": "^1.6.0" } }, "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg=="],
|
||||
|
||||
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="],
|
||||
|
||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
|
||||
"is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="],
|
||||
|
||||
"is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="],
|
||||
@@ -110,7 +126,7 @@
|
||||
|
||||
"is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="],
|
||||
|
||||
"jsdom": ["jsdom@26.1.0", "", { "dependencies": { "cssstyle": "^4.2.1", "data-urls": "^5.0.0", "decimal.js": "^10.5.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.16", "parse5": "^7.2.1", "rrweb-cssom": "^0.8.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^5.1.1", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.1.1", "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg=="],
|
||||
"jsdom": ["jsdom@29.0.2", "", { "dependencies": { "@asamuzakjp/css-color": "^5.1.5", "@asamuzakjp/dom-selector": "^7.0.6", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.1", "@exodus/bytes": "^1.15.0", "css-tree": "^3.2.1", "data-urls": "^7.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "is-potential-custom-element-name": "^1.0.1", "lru-cache": "^11.2.7", "parse5": "^8.0.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.1", "undici": "^7.24.5", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.1", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w=="],
|
||||
|
||||
"lighthouse-logger": ["lighthouse-logger@2.0.2", "", { "dependencies": { "debug": "^4.4.1", "marky": "^1.2.2" } }, "sha512-vWl2+u5jgOQuZR55Z1WM0XDdrJT6mzMP8zHUct7xTlWhuQs+eV0g+QL0RQdFjT54zVmbhLCP8vIVpy1wGn/gCg=="],
|
||||
|
||||
@@ -118,7 +134,7 @@
|
||||
|
||||
"longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="],
|
||||
|
||||
"lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
||||
"lru-cache": ["lru-cache@11.3.5", "", {}, "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw=="],
|
||||
|
||||
"markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="],
|
||||
|
||||
@@ -148,6 +164,8 @@
|
||||
|
||||
"mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],
|
||||
|
||||
"mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="],
|
||||
|
||||
"micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="],
|
||||
|
||||
"micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="],
|
||||
@@ -208,9 +226,7 @@
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"nwsapi": ["nwsapi@2.2.23", "", {}, "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ=="],
|
||||
|
||||
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||
"parse5": ["parse5@8.0.1", "", { "dependencies": { "entities": "^8.0.0" } }, "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw=="],
|
||||
|
||||
"punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
|
||||
|
||||
@@ -220,23 +236,23 @@
|
||||
|
||||
"remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="],
|
||||
|
||||
"rrweb-cssom": ["rrweb-cssom@0.8.0", "", {}, "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
"require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
|
||||
|
||||
"saxes": ["saxes@6.0.0", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA=="],
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
|
||||
"symbol-tree": ["symbol-tree@3.2.4", "", {}, "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="],
|
||||
|
||||
"temml": ["temml@0.13.2", "", {}, "sha512-n8fDRSsLscq9nh9j6z+FgkCvFMT0IJm6GCgwfzh+7AHT3Sfb4jFTQlsA6hVcF2dYYr3b66oDBVES95RfoukyrA=="],
|
||||
|
||||
"tldts": ["tldts@6.1.86", "", { "dependencies": { "tldts-core": "^6.1.86" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ=="],
|
||||
"tldts": ["tldts@7.0.28", "", { "dependencies": { "tldts-core": "^7.0.28" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw=="],
|
||||
|
||||
"tldts-core": ["tldts-core@6.1.86", "", {}, "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA=="],
|
||||
"tldts-core": ["tldts-core@7.0.28", "", {}, "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ=="],
|
||||
|
||||
"tough-cookie": ["tough-cookie@5.1.2", "", { "dependencies": { "tldts": "^6.1.32" } }, "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A=="],
|
||||
"tough-cookie": ["tough-cookie@6.0.1", "", { "dependencies": { "tldts": "^7.0.5" } }, "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw=="],
|
||||
|
||||
"tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="],
|
||||
"tr46": ["tr46@6.0.0", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw=="],
|
||||
|
||||
"trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="],
|
||||
|
||||
@@ -246,6 +262,8 @@
|
||||
|
||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||
|
||||
"undici": ["undici@7.25.0", "", {}, "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ=="],
|
||||
|
||||
"undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="],
|
||||
|
||||
"unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="],
|
||||
@@ -264,13 +282,11 @@
|
||||
|
||||
"w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="],
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="],
|
||||
"webidl-conversions": ["webidl-conversions@8.0.1", "", {}, "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
"whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
|
||||
"whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "^1.11.0", "tr46": "^6.0.0", "webidl-conversions": "^8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="],
|
||||
|
||||
"ws": ["ws@8.20.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA=="],
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { Adapter } from "../types";
|
||||
import { detectInteractionGate } from "../../browser/interaction-gates";
|
||||
import { captureNormalizedPageSnapshot } from "../../browser/page-snapshot";
|
||||
import { convertHtmlToMarkdown } from "../../extract/html-to-markdown";
|
||||
|
||||
export const genericAdapter: Adapter = {
|
||||
name: "generic",
|
||||
match() {
|
||||
return true;
|
||||
},
|
||||
async process(context) {
|
||||
context.log.info(`Loading ${context.input.url.toString()} with generic adapter`);
|
||||
await context.browser.goto(context.input.url.toString(), context.timeoutMs);
|
||||
|
||||
try {
|
||||
await context.network.waitForIdle({
|
||||
idleMs: 1_200,
|
||||
timeoutMs: Math.min(context.timeoutMs, 15_000),
|
||||
});
|
||||
} catch {
|
||||
context.log.debug("Network idle timed out on initial load; continuing.");
|
||||
}
|
||||
|
||||
await context.browser.scrollToEnd({ maxSteps: 4, delayMs: 300 });
|
||||
|
||||
try {
|
||||
await context.network.waitForIdle({
|
||||
idleMs: 900,
|
||||
timeoutMs: Math.min(context.timeoutMs, 10_000),
|
||||
});
|
||||
} catch {
|
||||
context.log.debug("Network idle timed out after scrolling; continuing.");
|
||||
}
|
||||
|
||||
const interaction = await detectInteractionGate(context.browser);
|
||||
if (interaction) {
|
||||
return {
|
||||
status: "needs_interaction",
|
||||
interaction,
|
||||
};
|
||||
}
|
||||
|
||||
const snapshot = await captureNormalizedPageSnapshot(context.browser);
|
||||
const converted = await convertHtmlToMarkdown(snapshot.html, snapshot.finalUrl, {
|
||||
enableRemoteMarkdownFallback: context.outputFormat === "markdown",
|
||||
preserveBase64Images: context.downloadMedia,
|
||||
});
|
||||
const document = {
|
||||
url: snapshot.finalUrl,
|
||||
canonicalUrl: converted.metadata.canonicalUrl,
|
||||
title: converted.metadata.title,
|
||||
author: converted.metadata.author,
|
||||
siteName: converted.metadata.siteName,
|
||||
publishedAt: converted.metadata.publishedAt,
|
||||
summary: converted.metadata.summary,
|
||||
adapter: "generic",
|
||||
metadata: {
|
||||
coverImage: converted.metadata.coverImage,
|
||||
language: converted.metadata.language,
|
||||
capturedAt: converted.metadata.capturedAt,
|
||||
conversionMethod: converted.conversionMethod,
|
||||
fallbackReason: converted.fallbackReason,
|
||||
kind: "generic/article",
|
||||
},
|
||||
content: converted.markdown ? [{ type: "markdown" as const, markdown: converted.markdown }] : [],
|
||||
};
|
||||
|
||||
return {
|
||||
status: "ok",
|
||||
document,
|
||||
media: converted.media,
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,391 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
import type { Adapter } from "../types";
|
||||
import type { ExtractedDocument } from "../../extract/document";
|
||||
import { collectMediaFromDocument } from "../../media/markdown-media";
|
||||
|
||||
const HN_BASE_URL = "https://news.ycombinator.com";
|
||||
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: "atx",
|
||||
bulletListMarker: "-",
|
||||
codeBlockStyle: "fenced",
|
||||
});
|
||||
|
||||
turndown.use(gfm);
|
||||
|
||||
export interface HnItem {
|
||||
id: number;
|
||||
type: "story" | "comment" | "job" | "poll" | "pollopt" | string;
|
||||
by?: string;
|
||||
time?: number;
|
||||
text?: string;
|
||||
title?: string;
|
||||
url?: string;
|
||||
score?: number;
|
||||
descendants?: number;
|
||||
kids?: number[];
|
||||
parent?: number;
|
||||
deleted?: boolean;
|
||||
dead?: boolean;
|
||||
}
|
||||
|
||||
export interface HnCommentNode {
|
||||
item: HnItem;
|
||||
children: HnCommentNode[];
|
||||
}
|
||||
|
||||
interface ParsedHnThread {
|
||||
story: HnItem;
|
||||
comments: HnCommentNode[];
|
||||
}
|
||||
|
||||
function decodeHtmlText(value: string | undefined): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const dom = new JSDOM(`<!doctype html><html><body>${value}</body></html>`);
|
||||
return dom.window.document.body.textContent?.trim() || undefined;
|
||||
}
|
||||
|
||||
function normalizeMarkdown(markdown: string): string {
|
||||
return markdown
|
||||
.replace(/\r\n/g, "\n")
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function convertHnHtmlToMarkdown(html: string | undefined, baseUrl: string): string {
|
||||
if (!html?.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const dom = new JSDOM(`<div id="__root">${html}</div>`, { url: baseUrl });
|
||||
const root = dom.window.document.querySelector("#__root");
|
||||
if (!root) {
|
||||
return "";
|
||||
}
|
||||
|
||||
root.querySelectorAll("a[href]").forEach((element) => {
|
||||
const href = element.getAttribute("href");
|
||||
if (!href) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
element.setAttribute("href", new URL(href, baseUrl).toString());
|
||||
} catch {
|
||||
// Ignore malformed URLs and keep the original href.
|
||||
}
|
||||
});
|
||||
|
||||
return normalizeMarkdown(turndown.turndown(root.innerHTML));
|
||||
}
|
||||
|
||||
function formatIsoTimestamp(unixSeconds: number | undefined): string | undefined {
|
||||
if (!unixSeconds || !Number.isFinite(unixSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return new Date(unixSeconds * 1_000).toISOString();
|
||||
}
|
||||
|
||||
function formatDisplayTimestamp(unixSeconds: number | undefined): string {
|
||||
const iso = formatIsoTimestamp(unixSeconds);
|
||||
if (!iso) {
|
||||
return "unknown time";
|
||||
}
|
||||
|
||||
return iso.replace("T", " ").replace(".000Z", " UTC");
|
||||
}
|
||||
|
||||
function indentMarkdown(markdown: string, spaces: number): string {
|
||||
const prefix = " ".repeat(spaces);
|
||||
return markdown
|
||||
.split("\n")
|
||||
.map((line) => (line ? `${prefix}${line}` : prefix))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function renderCommentHeader(item: HnItem, pageUrl: string): string {
|
||||
const author = item.by ?? "[deleted]";
|
||||
const time = item.id
|
||||
? `[${formatDisplayTimestamp(item.time)}](${pageUrl}#${item.id})`
|
||||
: formatDisplayTimestamp(item.time);
|
||||
return `${author} · ${time}`;
|
||||
}
|
||||
|
||||
function renderCommentNode(node: HnCommentNode, pageUrl: string, depth = 0): string {
|
||||
const baseIndent = " ".repeat(depth * 4);
|
||||
const lines = [`${baseIndent}- ${renderCommentHeader(node.item, pageUrl)}`];
|
||||
const body = convertHnHtmlToMarkdown(node.item.text, pageUrl);
|
||||
|
||||
if (body) {
|
||||
lines.push("");
|
||||
lines.push(indentMarkdown(body, depth * 4 + 4));
|
||||
} else if (node.item.deleted || node.item.dead) {
|
||||
lines.push("");
|
||||
lines.push(`${baseIndent} [comment unavailable]`);
|
||||
}
|
||||
|
||||
for (const child of node.children) {
|
||||
lines.push("");
|
||||
lines.push(renderCommentNode(child, pageUrl, depth + 1));
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function buildHnThreadMarkdown(
|
||||
story: HnItem,
|
||||
comments: HnCommentNode[],
|
||||
pageUrl: string,
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
const storyUrl = story.url ? new URL(story.url, pageUrl).toString() : undefined;
|
||||
const storyText = convertHnHtmlToMarkdown(story.text, pageUrl);
|
||||
|
||||
if (storyUrl && storyUrl !== pageUrl) {
|
||||
lines.push(`Source: [${storyUrl}](${storyUrl})`);
|
||||
}
|
||||
lines.push(`HN Item: [${story.id}](${pageUrl})`);
|
||||
|
||||
const submittedBy = story.by ? ` by ${story.by}` : "";
|
||||
const submittedAt = formatDisplayTimestamp(story.time);
|
||||
lines.push(`Submitted${submittedBy} at ${submittedAt}`);
|
||||
|
||||
const stats: string[] = [];
|
||||
if (typeof story.score === "number") {
|
||||
stats.push(`${story.score} points`);
|
||||
}
|
||||
if (typeof story.descendants === "number") {
|
||||
stats.push(`${story.descendants} comments`);
|
||||
}
|
||||
if (stats.length > 0) {
|
||||
lines.push(stats.join(" | "));
|
||||
}
|
||||
|
||||
if (storyText) {
|
||||
lines.push("");
|
||||
lines.push("## Post");
|
||||
lines.push("");
|
||||
lines.push(storyText);
|
||||
}
|
||||
|
||||
lines.push("");
|
||||
lines.push("## Comments");
|
||||
lines.push("");
|
||||
|
||||
if (comments.length === 0) {
|
||||
lines.push("No comments.");
|
||||
} else {
|
||||
lines.push(comments.map((comment) => renderCommentNode(comment, pageUrl)).join("\n\n"));
|
||||
}
|
||||
|
||||
return normalizeMarkdown(lines.join("\n"));
|
||||
}
|
||||
|
||||
export function buildHnDocument(
|
||||
story: HnItem,
|
||||
comments: HnCommentNode[],
|
||||
pageUrl: string,
|
||||
): ExtractedDocument {
|
||||
const decodedTitle = decodeHtmlText(story.title) ?? `HN Item ${story.id}`;
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
canonicalUrl: pageUrl,
|
||||
title: decodedTitle,
|
||||
author: story.by,
|
||||
siteName: "Hacker News",
|
||||
publishedAt: formatIsoTimestamp(story.time),
|
||||
adapter: "hn",
|
||||
metadata: {
|
||||
kind: "hn/story",
|
||||
storyId: story.id,
|
||||
storyUrl: story.url ? new URL(story.url, pageUrl).toString() : undefined,
|
||||
points: story.score,
|
||||
commentCount: story.descendants,
|
||||
},
|
||||
content: [
|
||||
{
|
||||
type: "markdown",
|
||||
markdown: buildHnThreadMarkdown(story, comments, pageUrl),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function parseHnItemId(url: URL): number | null {
|
||||
if (url.hostname !== "news.ycombinator.com") {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (url.pathname !== "/item") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const value = url.searchParams.get("id");
|
||||
if (!value || !/^\d+$/.test(value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
function extractUnixSecondsFromAge(element: Element | null): number | undefined {
|
||||
const title = element?.getAttribute("title")?.trim();
|
||||
if (!title) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const match = title.match(/(\d{9,})$/);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
function extractScore(text: string | null | undefined): number | undefined {
|
||||
if (!text) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const match = text.match(/(\d+)/);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
function extractCommentCount(container: ParentNode): number | undefined {
|
||||
const anchors = Array.from(container.querySelectorAll("a"));
|
||||
for (const anchor of anchors) {
|
||||
const match = anchor.textContent?.trim().match(/(\d+)\s+comments?/i);
|
||||
if (match) {
|
||||
return Number(match[1]);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeStoryUrl(storyId: number, href: string | null | undefined, pageUrl: string): string | undefined {
|
||||
if (!href) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const resolved = new URL(href, pageUrl).toString();
|
||||
if (resolved === pageUrl || resolved === `${HN_BASE_URL}/item?id=${storyId}`) {
|
||||
return undefined;
|
||||
}
|
||||
return resolved;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractHnThreadFromHtml(html: string, pageUrl: string): ParsedHnThread | null {
|
||||
const dom = new JSDOM(html, { url: pageUrl });
|
||||
const { document } = dom.window;
|
||||
const storyRow = document.querySelector("table.fatitem tr.athing.submission");
|
||||
if (!storyRow) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const storyId = Number(storyRow.getAttribute("id"));
|
||||
if (!Number.isFinite(storyId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const titleLink = storyRow.querySelector(".titleline > a");
|
||||
const subline = document.querySelector("table.fatitem .subline");
|
||||
const topText = document.querySelector("table.fatitem .toptext");
|
||||
|
||||
const story: HnItem = {
|
||||
id: storyId,
|
||||
type: "story",
|
||||
by: subline?.querySelector(".hnuser")?.textContent?.trim() || undefined,
|
||||
time: extractUnixSecondsFromAge(subline?.querySelector(".age") ?? null),
|
||||
title: titleLink?.innerHTML?.trim() || undefined,
|
||||
url: normalizeStoryUrl(storyId, titleLink?.getAttribute("href"), pageUrl),
|
||||
text: topText?.innerHTML?.trim() || undefined,
|
||||
score: extractScore(subline?.querySelector(".score")?.textContent),
|
||||
descendants: extractCommentCount(subline ?? document),
|
||||
};
|
||||
|
||||
const roots: HnCommentNode[] = [];
|
||||
const stack: HnCommentNode[] = [];
|
||||
|
||||
document.querySelectorAll("tr.athing.comtr").forEach((row) => {
|
||||
const commentId = Number(row.getAttribute("id"));
|
||||
if (!Number.isFinite(commentId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const indentRaw = row.querySelector("td.ind")?.getAttribute("indent");
|
||||
const depth = indentRaw && /^\d+$/.test(indentRaw) ? Number(indentRaw) : 0;
|
||||
const comhead = row.querySelector(".comhead");
|
||||
const item: HnItem = {
|
||||
id: commentId,
|
||||
type: "comment",
|
||||
by: comhead?.querySelector(".hnuser")?.textContent?.trim() || undefined,
|
||||
time: extractUnixSecondsFromAge(comhead?.querySelector(".age") ?? null),
|
||||
text: row.querySelector(".comment > .commtext")?.innerHTML?.trim() || undefined,
|
||||
deleted: row.querySelector(".comment > .commtext") === null,
|
||||
};
|
||||
|
||||
const node: HnCommentNode = {
|
||||
item,
|
||||
children: [],
|
||||
};
|
||||
|
||||
while (stack.length > depth) {
|
||||
stack.pop();
|
||||
}
|
||||
|
||||
const parent = stack[stack.length - 1];
|
||||
if (parent) {
|
||||
parent.children.push(node);
|
||||
} else {
|
||||
roots.push(node);
|
||||
}
|
||||
|
||||
stack.push(node);
|
||||
});
|
||||
|
||||
return {
|
||||
story,
|
||||
comments: roots,
|
||||
};
|
||||
}
|
||||
|
||||
export const hnAdapter: Adapter = {
|
||||
name: "hn",
|
||||
match(input) {
|
||||
return parseHnItemId(input.url) !== null;
|
||||
},
|
||||
async process(context) {
|
||||
const itemId = parseHnItemId(context.input.url);
|
||||
if (!itemId) {
|
||||
return {
|
||||
status: "no_document",
|
||||
};
|
||||
}
|
||||
|
||||
const pageUrl = context.input.url.toString();
|
||||
context.log.info(`Loading ${pageUrl} with hn adapter`);
|
||||
await context.browser.goto(pageUrl, context.timeoutMs);
|
||||
const html = await context.browser.getHTML();
|
||||
const thread = extractHnThreadFromHtml(html, pageUrl);
|
||||
if (!thread) {
|
||||
return {
|
||||
status: "no_document",
|
||||
};
|
||||
}
|
||||
|
||||
const document = buildHnDocument(thread.story, thread.comments, pageUrl);
|
||||
return {
|
||||
status: "ok",
|
||||
document,
|
||||
media: collectMediaFromDocument(document),
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { Adapter, AdapterInput } from "./types";
|
||||
import { genericAdapter } from "./generic";
|
||||
import { hnAdapter } from "./hn";
|
||||
import { xAdapter } from "./x";
|
||||
import { youtubeAdapter } from "./youtube";
|
||||
|
||||
const adapters: Adapter[] = [xAdapter, youtubeAdapter, hnAdapter, genericAdapter];
|
||||
|
||||
export function listAdapters(): Adapter[] {
|
||||
return adapters;
|
||||
}
|
||||
|
||||
export function resolveAdapter(input: AdapterInput, forcedName?: string): Adapter {
|
||||
if (forcedName) {
|
||||
const forced = adapters.find((adapter) => adapter.name === forcedName);
|
||||
if (!forced) {
|
||||
throw new Error(`Unknown adapter: ${forcedName}`);
|
||||
}
|
||||
return forced;
|
||||
}
|
||||
|
||||
const matched = adapters.find((adapter) => adapter.match(input));
|
||||
if (!matched) {
|
||||
throw new Error("No adapter matched the URL");
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
export { genericAdapter };
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { BrowserSession } from "../browser/session";
|
||||
import type { CdpClient } from "../browser/cdp-client";
|
||||
import type { NetworkJournal } from "../browser/network-journal";
|
||||
import type { ExtractedDocument } from "../extract/document";
|
||||
import type { MediaDownloadRequest, MediaDownloadResult, MediaAsset } from "../media/types";
|
||||
import type { Logger } from "../utils/logger";
|
||||
|
||||
export interface AdapterInput {
|
||||
url: URL;
|
||||
}
|
||||
|
||||
export type LoginState = "logged_in" | "logged_out" | "unknown";
|
||||
export type InteractionKind = "login" | "cloudflare" | "recaptcha" | "hcaptcha" | "captcha" | "challenge";
|
||||
|
||||
export interface AdapterLoginInfo {
|
||||
provider: string;
|
||||
state: LoginState;
|
||||
required?: boolean;
|
||||
username?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface WaitForInteractionRequest {
|
||||
type: "wait_for_interaction";
|
||||
kind: InteractionKind;
|
||||
provider: string;
|
||||
prompt: string;
|
||||
reason?: string;
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
requiresVisibleBrowser?: boolean;
|
||||
}
|
||||
|
||||
export type AdapterProcessResult =
|
||||
| {
|
||||
status: "ok";
|
||||
document: ExtractedDocument;
|
||||
media?: MediaAsset[];
|
||||
login?: AdapterLoginInfo;
|
||||
}
|
||||
| {
|
||||
status: "needs_interaction";
|
||||
interaction: WaitForInteractionRequest;
|
||||
login?: AdapterLoginInfo;
|
||||
}
|
||||
| {
|
||||
status: "no_document";
|
||||
login?: AdapterLoginInfo;
|
||||
};
|
||||
|
||||
export interface AdapterContext {
|
||||
input: AdapterInput;
|
||||
browser: BrowserSession;
|
||||
network: NetworkJournal;
|
||||
cdp: CdpClient;
|
||||
log: Logger;
|
||||
outputFormat: "markdown" | "json";
|
||||
timeoutMs: number;
|
||||
interactive: boolean;
|
||||
downloadMedia: boolean;
|
||||
}
|
||||
|
||||
export interface Adapter {
|
||||
name: string;
|
||||
match(input: AdapterInput): boolean;
|
||||
checkLogin?(context: AdapterContext): Promise<AdapterLoginInfo>;
|
||||
exportCookies?(context: AdapterContext, profileDir?: string): Promise<boolean>;
|
||||
restoreCookies?(context: AdapterContext, profileDir?: string): Promise<boolean>;
|
||||
downloadMedia?(request: MediaDownloadRequest): Promise<MediaDownloadResult>;
|
||||
process(context: AdapterContext): Promise<AdapterProcessResult>;
|
||||
}
|
||||
|
||||
export type { MediaAsset };
|
||||
@@ -0,0 +1,461 @@
|
||||
import type { ExtractedDocument } from "../../extract/document";
|
||||
import {
|
||||
findTweetNode,
|
||||
findTweetNodeById,
|
||||
formatMediaList,
|
||||
formatTweetAuthor,
|
||||
getTweetAuthorMetadata,
|
||||
getTweetText,
|
||||
getUser,
|
||||
isRecord,
|
||||
normalizeTitle,
|
||||
resolveBestXVideoVariantUrl,
|
||||
toHighResXImageUrl,
|
||||
toXTweet,
|
||||
} from "./shared";
|
||||
import type { JsonObject } from "./types";
|
||||
|
||||
interface ArticleMedia {
|
||||
kind: "image" | "video";
|
||||
url: string;
|
||||
}
|
||||
|
||||
function resolveArticleMedia(mediaInfo: JsonObject): ArticleMedia | null {
|
||||
const videoUrl = resolveBestXVideoVariantUrl(mediaInfo);
|
||||
if (videoUrl) {
|
||||
return {
|
||||
kind: "video",
|
||||
url: videoUrl,
|
||||
};
|
||||
}
|
||||
|
||||
const rawUrl =
|
||||
(typeof mediaInfo.original_img_url === "string" && mediaInfo.original_img_url) ||
|
||||
(typeof mediaInfo.url === "string" && mediaInfo.url) ||
|
||||
"";
|
||||
|
||||
if (!rawUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "image",
|
||||
url: toHighResXImageUrl(rawUrl),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveArticleMediaUrl(mediaInfo: JsonObject): string {
|
||||
return resolveArticleMedia(mediaInfo)?.url ?? "";
|
||||
}
|
||||
|
||||
function normalizeEntityMap(entityMap: unknown): Map<string, JsonObject> {
|
||||
const normalized = new Map<string, JsonObject>();
|
||||
|
||||
if (Array.isArray(entityMap)) {
|
||||
for (const entry of entityMap) {
|
||||
if (!isRecord(entry)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key =
|
||||
typeof entry.key === "string" || typeof entry.key === "number"
|
||||
? String(entry.key)
|
||||
: undefined;
|
||||
const value = isRecord(entry.value) ? entry.value : undefined;
|
||||
if (!key || !value) {
|
||||
continue;
|
||||
}
|
||||
normalized.set(key, value);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (!isRecord(entityMap)) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(entityMap)) {
|
||||
if (!isRecord(value)) {
|
||||
continue;
|
||||
}
|
||||
normalized.set(key, value);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function getEntityMarkdown(entityMap: Map<string, JsonObject>, entityKey: unknown): string | null {
|
||||
const key =
|
||||
typeof entityKey === "string" || typeof entityKey === "number"
|
||||
? String(entityKey)
|
||||
: undefined;
|
||||
if (!key) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entity = entityMap.get(key);
|
||||
if (!entity || entity.type !== "MARKDOWN") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = isRecord(entity.data) ? entity.data : {};
|
||||
if (typeof data.markdown !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const markdown = data.markdown.trim();
|
||||
return markdown || null;
|
||||
}
|
||||
|
||||
function getLinkUrl(entityMap: Map<string, JsonObject>, entityKey: unknown): string | null {
|
||||
const key =
|
||||
typeof entityKey === "string" || typeof entityKey === "number"
|
||||
? String(entityKey)
|
||||
: undefined;
|
||||
if (!key) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entity = entityMap.get(key);
|
||||
if (!entity || entity.type !== "LINK") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = isRecord(entity.data) ? entity.data : {};
|
||||
const candidates = [
|
||||
data.expanded_url,
|
||||
data.expandedUrl,
|
||||
data.original_url,
|
||||
data.originalUrl,
|
||||
data.url,
|
||||
data.display_url,
|
||||
data.displayUrl,
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate === "string" && candidate.trim()) {
|
||||
return candidate.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function getTweetId(entityMap: Map<string, JsonObject>, entityKey: unknown): string | null {
|
||||
const key =
|
||||
typeof entityKey === "string" || typeof entityKey === "number"
|
||||
? String(entityKey)
|
||||
: undefined;
|
||||
if (!key) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entity = entityMap.get(key);
|
||||
if (!entity || entity.type !== "TWEET") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = isRecord(entity.data) ? entity.data : {};
|
||||
if (typeof data.tweetId !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return data.tweetId;
|
||||
}
|
||||
|
||||
function buildMediaMap(articleResult: JsonObject): Map<string, ArticleMedia> {
|
||||
const mediaMap = new Map<string, ArticleMedia>();
|
||||
const mediaEntities = Array.isArray(articleResult.media_entities) ? articleResult.media_entities : [];
|
||||
|
||||
for (const entity of mediaEntities) {
|
||||
if (!isRecord(entity) || typeof entity.media_id !== "string" || !isRecord(entity.media_info)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const media = resolveArticleMedia(entity.media_info);
|
||||
if (media) {
|
||||
mediaMap.set(entity.media_id, media);
|
||||
}
|
||||
}
|
||||
|
||||
const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null;
|
||||
if (coverMedia && typeof coverMedia.media_id === "string" && isRecord(coverMedia.media_info)) {
|
||||
const media = resolveArticleMedia(coverMedia.media_info);
|
||||
if (media) {
|
||||
mediaMap.set(coverMedia.media_id, media);
|
||||
}
|
||||
}
|
||||
|
||||
return mediaMap;
|
||||
}
|
||||
|
||||
function getMediaMarkdown(
|
||||
entityMap: Map<string, JsonObject>,
|
||||
entityKey: unknown,
|
||||
mediaMap: Map<string, ArticleMedia>,
|
||||
): string[] {
|
||||
const key =
|
||||
typeof entityKey === "string" || typeof entityKey === "number"
|
||||
? String(entityKey)
|
||||
: undefined;
|
||||
if (!key) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const entity = entityMap.get(key);
|
||||
if (!entity || entity.type !== "MEDIA") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const data = isRecord(entity.data) ? entity.data : {};
|
||||
const mediaItems = Array.isArray(data.mediaItems) ? data.mediaItems : [];
|
||||
const media: ArticleMedia[] = [];
|
||||
|
||||
for (const item of mediaItems) {
|
||||
if (!isRecord(item) || typeof item.mediaId !== "string") {
|
||||
continue;
|
||||
}
|
||||
const mediaItem = mediaMap.get(item.mediaId);
|
||||
if (mediaItem && !media.some((value) => value.url === mediaItem.url)) {
|
||||
media.push(mediaItem);
|
||||
}
|
||||
}
|
||||
|
||||
return media.map((item) => item.kind === "image" ? `` : `[video](${item.url})`);
|
||||
}
|
||||
|
||||
function resolveTweetMarkdown(payloads: unknown[], tweetId: string, pageUrl: string): string | null {
|
||||
for (const payload of payloads) {
|
||||
const tweet = findTweetNodeById(payload, tweetId);
|
||||
if (!tweet) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const xTweet = toXTweet(tweet, pageUrl);
|
||||
const author = formatTweetAuthor(xTweet) ?? xTweet.url;
|
||||
const lines = [`> ${author}`, ...xTweet.text.split("\n").map((line) => `> ${line}`)];
|
||||
|
||||
const media = formatMediaList(xTweet.media).map((line) =>
|
||||
line.startsWith("photo: ") ? `> })` : `> - ${line}`,
|
||||
);
|
||||
|
||||
const parts = [lines.join("\n")];
|
||||
if (media.length > 0) {
|
||||
parts.push([">", ...media].join("\n"));
|
||||
}
|
||||
parts.push(`> ${xTweet.url}`);
|
||||
|
||||
return parts.join("\n").trim();
|
||||
}
|
||||
|
||||
return `> Embedded tweet: https://x.com/i/status/${tweetId}`;
|
||||
}
|
||||
|
||||
function replaceLinkEntities(text: string, block: JsonObject, entityMap: Map<string, JsonObject>): string {
|
||||
const entityRanges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
|
||||
const replacements = entityRanges
|
||||
.filter((range): range is JsonObject => isRecord(range))
|
||||
.map((range) => {
|
||||
const offset = typeof range.offset === "number" ? range.offset : -1;
|
||||
const length = typeof range.length === "number" ? range.length : -1;
|
||||
const url = getLinkUrl(entityMap, range.key);
|
||||
return { offset, length, url };
|
||||
})
|
||||
.filter((range) => range.offset >= 0 && range.length > 0 && range.url)
|
||||
.sort((left, right) => right.offset - left.offset);
|
||||
|
||||
let next = text;
|
||||
for (const replacement of replacements) {
|
||||
next =
|
||||
next.slice(0, replacement.offset) +
|
||||
replacement.url +
|
||||
next.slice(replacement.offset + replacement.length);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function renderAtomicBlock(
|
||||
block: JsonObject,
|
||||
entityMap: Map<string, JsonObject>,
|
||||
mediaMap: Map<string, ArticleMedia>,
|
||||
payloads: unknown[],
|
||||
pageUrl: string,
|
||||
): string | null {
|
||||
const entityRanges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
|
||||
const parts: string[] = [];
|
||||
|
||||
for (const range of entityRanges) {
|
||||
if (!isRecord(range)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const markdown = getEntityMarkdown(entityMap, range.key);
|
||||
if (markdown) {
|
||||
parts.push(markdown);
|
||||
continue;
|
||||
}
|
||||
|
||||
const mediaMarkdown = getMediaMarkdown(entityMap, range.key, mediaMap);
|
||||
if (mediaMarkdown.length > 0) {
|
||||
parts.push(mediaMarkdown.join("\n\n"));
|
||||
continue;
|
||||
}
|
||||
|
||||
const tweetId = getTweetId(entityMap, range.key);
|
||||
if (tweetId) {
|
||||
const tweetMarkdown = resolveTweetMarkdown(payloads, tweetId, pageUrl);
|
||||
if (tweetMarkdown) {
|
||||
parts.push(tweetMarkdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return parts.join("\n\n");
|
||||
}
|
||||
|
||||
function renderArticleBlocks(
|
||||
blocks: unknown[],
|
||||
entityMap: Map<string, JsonObject>,
|
||||
mediaMap: Map<string, ArticleMedia>,
|
||||
payloads: unknown[],
|
||||
pageUrl: string,
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
let orderedCounter = 0;
|
||||
|
||||
for (const block of blocks) {
|
||||
if (!isRecord(block)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const blockType = typeof block.type === "string" ? block.type : "unstyled";
|
||||
const rawText = typeof block.text === "string" ? block.text : "";
|
||||
const text = replaceLinkEntities(rawText, block, entityMap).trim();
|
||||
if (!text && blockType !== "atomic") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (blockType !== "ordered-list-item") {
|
||||
orderedCounter = 0;
|
||||
}
|
||||
|
||||
switch (blockType) {
|
||||
case "header-one":
|
||||
parts.push(`# ${text}`);
|
||||
break;
|
||||
case "header-two":
|
||||
parts.push(`## ${text}`);
|
||||
break;
|
||||
case "header-three":
|
||||
parts.push(`### ${text}`);
|
||||
break;
|
||||
case "blockquote":
|
||||
parts.push(`> ${text}`);
|
||||
break;
|
||||
case "unordered-list-item":
|
||||
parts.push(`- ${text}`);
|
||||
break;
|
||||
case "ordered-list-item":
|
||||
orderedCounter += 1;
|
||||
parts.push(`${orderedCounter}. ${text}`);
|
||||
break;
|
||||
case "code-block":
|
||||
parts.push(`\`\`\`\n${text}\n\`\`\``);
|
||||
break;
|
||||
case "atomic": {
|
||||
const markdown = renderAtomicBlock(block, entityMap, mediaMap, payloads, pageUrl);
|
||||
if (markdown) {
|
||||
parts.push(markdown);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
parts.push(text);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join("\n\n").trim();
|
||||
}
|
||||
|
||||
function getArticleResult(tweet: JsonObject): JsonObject | null {
|
||||
if (
|
||||
isRecord(tweet.article) &&
|
||||
isRecord(tweet.article.article_results) &&
|
||||
isRecord(tweet.article.article_results.result)
|
||||
) {
|
||||
return tweet.article.article_results.result as JsonObject;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractSummary(markdown: string): string | undefined {
|
||||
const segments = markdown
|
||||
.split(/\n\n+/)
|
||||
.map((segment) => segment.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const preferred = segments.find((segment) => !/^(#|>|- |\d+\. |\`\`\`)/.test(segment));
|
||||
return preferred?.slice(0, 220);
|
||||
}
|
||||
|
||||
export function extractArticleDocumentFromPayload(
|
||||
payload: unknown,
|
||||
statusId: string,
|
||||
pageUrl: string,
|
||||
payloads: unknown[] = [payload],
|
||||
): ExtractedDocument | null {
|
||||
const tweet = findTweetNode(payload, statusId);
|
||||
if (!tweet) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const articleResult = getArticleResult(tweet);
|
||||
if (!articleResult) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const title = typeof articleResult.title === "string" ? articleResult.title.trim() : undefined;
|
||||
const contentState = isRecord(articleResult.content_state) ? articleResult.content_state : {};
|
||||
const blocks = Array.isArray(contentState.blocks) ? contentState.blocks : [];
|
||||
const entityMap = normalizeEntityMap(contentState.entityMap);
|
||||
const mediaMap = buildMediaMap(articleResult);
|
||||
const richMarkdown = renderArticleBlocks(blocks, entityMap, mediaMap, payloads, pageUrl);
|
||||
const plainText = typeof articleResult.plain_text === "string" ? articleResult.plain_text.trim() : "";
|
||||
const markdown = richMarkdown || plainText || getTweetText(tweet);
|
||||
if (!markdown) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const xTweet = toXTweet(tweet, pageUrl);
|
||||
const user = getUser(tweet);
|
||||
const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null;
|
||||
const coverMediaInfo = coverMedia && isRecord(coverMedia.media_info) ? coverMedia.media_info : null;
|
||||
const coverImage = coverMediaInfo ? resolveArticleMediaUrl(coverMediaInfo) || undefined : undefined;
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
canonicalUrl: xTweet.url,
|
||||
title: title || normalizeTitle(xTweet.text, "X Article"),
|
||||
author: formatTweetAuthor(xTweet),
|
||||
siteName: "X",
|
||||
publishedAt: xTweet.createdAt,
|
||||
summary: extractSummary(markdown) || xTweet.text.slice(0, 200) || undefined,
|
||||
adapter: "x",
|
||||
metadata: {
|
||||
kind: "x/article",
|
||||
tweetId: xTweet.id,
|
||||
coverImage,
|
||||
authorName: xTweet.authorName ?? user.name,
|
||||
authorUsername: xTweet.author ?? user.screenName,
|
||||
authorUrl: (xTweet.author ?? user.screenName) ? `https://x.com/${xTweet.author ?? user.screenName}` : undefined,
|
||||
...getTweetAuthorMetadata(xTweet),
|
||||
},
|
||||
content: [{ type: "markdown", markdown }],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import type { Adapter, AdapterLoginInfo } from "../types";
|
||||
import { exportCookies, restoreCookies, type CookieSidecarConfig } from "../../browser/cookie-sidecar";
|
||||
import { detectInteractionGate } from "../../browser/interaction-gates";
|
||||
import type { ExtractedDocument } from "../../extract/document";
|
||||
import { collectMediaFromDocument } from "../../media/markdown-media";
|
||||
import { extractArticleDocumentFromPayload } from "./article";
|
||||
import { buildNeedsLoginResult, detectXLogin } from "./login";
|
||||
import { extractStatusId, isXHost } from "./match";
|
||||
import { collectXJsonPayloads, waitForInitialXPayload } from "./payloads";
|
||||
import { extractSingleTweetDocumentFromPayload } from "./single";
|
||||
import { extractThreadDocumentFromPayloads } from "./thread";
|
||||
import { loadFullXThread } from "./thread-loader";
|
||||
|
||||
const cookieConfig: CookieSidecarConfig = {
|
||||
urls: ["https://x.com/", "https://twitter.com/"],
|
||||
filename: "x-session-cookies.json",
|
||||
requiredCookieNames: ["auth_token", "ct0"],
|
||||
filterCookie: (c) => {
|
||||
const d = c.domain ?? "";
|
||||
return d.endsWith("x.com") || d.endsWith("twitter.com");
|
||||
},
|
||||
};
|
||||
|
||||
function extractDocumentFromPayloads(
|
||||
payloads: unknown[],
|
||||
statusId: string,
|
||||
pageUrl: string,
|
||||
): ExtractedDocument | null {
|
||||
for (const payload of payloads) {
|
||||
const articleDocument = extractArticleDocumentFromPayload(payload, statusId, pageUrl, payloads);
|
||||
if (articleDocument) {
|
||||
return articleDocument;
|
||||
}
|
||||
}
|
||||
|
||||
const threadDocument = extractThreadDocumentFromPayloads(payloads, statusId, pageUrl);
|
||||
if (threadDocument) {
|
||||
return threadDocument;
|
||||
}
|
||||
|
||||
for (const payload of payloads) {
|
||||
const singleDocument = extractSingleTweetDocumentFromPayload(payload, statusId, pageUrl);
|
||||
if (singleDocument) {
|
||||
return singleDocument;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function ensureXLoginState(context: Parameters<Adapter["process"]>[0]): Promise<AdapterLoginInfo> {
|
||||
return detectXLogin(context);
|
||||
}
|
||||
|
||||
export const xAdapter: Adapter = {
|
||||
name: "x",
|
||||
match(input) {
|
||||
return isXHost(input.url.hostname);
|
||||
},
|
||||
async checkLogin(context) {
|
||||
return detectXLogin(context);
|
||||
},
|
||||
async exportCookies(context, profileDir) {
|
||||
return exportCookies(context.browser.targetSession, cookieConfig, profileDir);
|
||||
},
|
||||
async restoreCookies(context, profileDir) {
|
||||
return restoreCookies(context.browser.targetSession, cookieConfig, profileDir);
|
||||
},
|
||||
async process(context) {
|
||||
const statusId = extractStatusId(context.input.url);
|
||||
if (!statusId) {
|
||||
return {
|
||||
status: "no_document",
|
||||
};
|
||||
}
|
||||
|
||||
context.log.info(`Loading ${context.input.url.toString()} with x adapter`);
|
||||
await context.browser.goto(context.input.url.toString(), context.timeoutMs);
|
||||
|
||||
const interaction = await detectInteractionGate(context.browser);
|
||||
if (interaction) {
|
||||
return {
|
||||
status: "needs_interaction",
|
||||
interaction,
|
||||
};
|
||||
}
|
||||
|
||||
let login = await ensureXLoginState(context);
|
||||
if (login.state === "logged_out") {
|
||||
return buildNeedsLoginResult(login);
|
||||
}
|
||||
|
||||
await waitForInitialXPayload(context);
|
||||
await loadFullXThread(context, statusId);
|
||||
|
||||
const pageUrl = await context.browser.getURL();
|
||||
const postLoadInteraction = await detectInteractionGate(context.browser);
|
||||
if (postLoadInteraction) {
|
||||
return {
|
||||
status: "needs_interaction",
|
||||
interaction: postLoadInteraction,
|
||||
login,
|
||||
};
|
||||
}
|
||||
|
||||
login = await ensureXLoginState(context).catch(() => login);
|
||||
if (login.state === "logged_out") {
|
||||
return buildNeedsLoginResult(login);
|
||||
}
|
||||
|
||||
const payloads = await collectXJsonPayloads(context);
|
||||
if (payloads.length === 0) {
|
||||
return {
|
||||
status: "no_document",
|
||||
login,
|
||||
};
|
||||
}
|
||||
|
||||
const document = extractDocumentFromPayloads(payloads, statusId, pageUrl);
|
||||
if (document) {
|
||||
return {
|
||||
status: "ok",
|
||||
document,
|
||||
media: collectMediaFromDocument(document),
|
||||
login,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "no_document",
|
||||
login,
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { AdapterContext, AdapterLoginInfo, AdapterProcessResult } from "../types";
|
||||
|
||||
interface XLoginSnapshot {
|
||||
currentUrl: string;
|
||||
hasAccountMenu: boolean;
|
||||
hasLoginInputs: boolean;
|
||||
bodyText: string;
|
||||
}
|
||||
|
||||
export async function detectXLogin(context: AdapterContext): Promise<AdapterLoginInfo> {
|
||||
const snapshot = await context.browser.evaluate<XLoginSnapshot>(`
|
||||
(() => {
|
||||
const bodyText = (document.body?.innerText ?? "").slice(0, 2500);
|
||||
return {
|
||||
currentUrl: window.location.href,
|
||||
hasAccountMenu: Boolean(
|
||||
document.querySelector(
|
||||
'[data-testid="SideNav_AccountSwitcher_Button"], [data-testid="AppTabBar_Profile_Link"], [aria-label="Account menu"]'
|
||||
)
|
||||
),
|
||||
hasLoginInputs: Boolean(
|
||||
document.querySelector(
|
||||
'input[name="text"], input[name="password"], input[autocomplete="username"], input[autocomplete="current-password"]'
|
||||
)
|
||||
),
|
||||
bodyText,
|
||||
};
|
||||
})()
|
||||
`).catch(async () => ({
|
||||
currentUrl: await context.browser.getURL().catch(() => context.input.url.toString()),
|
||||
hasAccountMenu: false,
|
||||
hasLoginInputs: false,
|
||||
bodyText: "",
|
||||
}));
|
||||
|
||||
if (
|
||||
/\/i\/flow\/login|\/login/i.test(snapshot.currentUrl) ||
|
||||
snapshot.hasLoginInputs ||
|
||||
/sign in to x|join x today|登录 x|注册 x|登录到 x/i.test(snapshot.bodyText)
|
||||
) {
|
||||
return {
|
||||
provider: "x",
|
||||
state: "logged_out",
|
||||
required: true,
|
||||
reason: "X login page detected",
|
||||
};
|
||||
}
|
||||
|
||||
if (snapshot.hasAccountMenu) {
|
||||
return {
|
||||
provider: "x",
|
||||
state: "logged_in",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "x",
|
||||
state: "unknown",
|
||||
};
|
||||
}
|
||||
|
||||
export function buildNeedsLoginResult(login: AdapterLoginInfo): AdapterProcessResult {
|
||||
return {
|
||||
status: "needs_interaction",
|
||||
login: {
|
||||
...login,
|
||||
provider: "x",
|
||||
state: login.state === "logged_in" ? "unknown" : login.state,
|
||||
required: true,
|
||||
},
|
||||
interaction: {
|
||||
type: "wait_for_interaction",
|
||||
kind: "login",
|
||||
provider: "x",
|
||||
reason: login.reason,
|
||||
prompt: "Please sign in to X in the opened Chrome window. Extraction will continue automatically once login is detected.",
|
||||
requiresVisibleBrowser: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export function isXHost(hostname: string): boolean {
|
||||
return ["x.com", "www.x.com", "twitter.com", "www.twitter.com"].includes(hostname);
|
||||
}
|
||||
|
||||
export function extractStatusId(url: URL): string | undefined {
|
||||
const match = url.pathname.match(/\/(?:status|article)\/(\d+)/);
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { AdapterContext } from "../types";
|
||||
import { filterXGraphQlEntries } from "./shared";
|
||||
|
||||
export function getRelevantXThreadEntries(context: AdapterContext) {
|
||||
return filterXGraphQlEntries(context.network.getEntries()).filter(
|
||||
(entry) =>
|
||||
entry.method === "GET" &&
|
||||
entry.finished &&
|
||||
(
|
||||
entry.url.includes("TweetDetail") ||
|
||||
entry.url.includes("TweetResultByRestId") ||
|
||||
entry.url.includes("TweetResultsByRestIds")
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function prefetchRelevantXThreadBodies(context: AdapterContext): Promise<void> {
|
||||
const entries = getRelevantXThreadEntries(context).filter((entry) => entry.body === undefined && !entry.bodyError);
|
||||
for (const entry of entries) {
|
||||
await context.network.ensureBody(entry);
|
||||
}
|
||||
}
|
||||
|
||||
export async function collectXJsonPayloads(context: AdapterContext): Promise<unknown[]> {
|
||||
await prefetchRelevantXThreadBodies(context);
|
||||
const entries = getRelevantXThreadEntries(context);
|
||||
|
||||
const payloads: unknown[] = [];
|
||||
for (const entry of entries) {
|
||||
const payload = await context.network.getJsonBody(entry);
|
||||
if (payload) {
|
||||
payloads.push(payload);
|
||||
}
|
||||
}
|
||||
return payloads;
|
||||
}
|
||||
|
||||
export async function waitForInitialXPayload(context: AdapterContext): Promise<void> {
|
||||
try {
|
||||
await context.network.waitForResponse(
|
||||
(entry) =>
|
||||
entry.url.includes("/graphql/") &&
|
||||
(entry.url.includes("TweetDetail") || entry.url.includes("TweetResultByRestId")),
|
||||
{ timeoutMs: Math.min(context.timeoutMs, 15_000) },
|
||||
);
|
||||
await prefetchRelevantXThreadBodies(context);
|
||||
} catch {
|
||||
context.log.debug("No tweet GraphQL response observed before timeout.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { AdapterContext } from "../types";
|
||||
|
||||
const X_SESSION_URLS = ["https://x.com/", "https://twitter.com/"] as const;
|
||||
const REQUIRED_X_SESSION_COOKIES = ["auth_token", "ct0"] as const;
|
||||
|
||||
interface CookieLike {
|
||||
name?: string;
|
||||
value?: string | null;
|
||||
}
|
||||
|
||||
interface NetworkGetCookiesResult {
|
||||
cookies?: CookieLike[];
|
||||
}
|
||||
|
||||
export function buildXSessionCookieMap(cookies: readonly CookieLike[]): Record<string, string> {
|
||||
const cookieMap: Record<string, string> = {};
|
||||
for (const cookie of cookies) {
|
||||
const name = cookie.name?.trim();
|
||||
const value = cookie.value?.trim();
|
||||
if (!name || !value) {
|
||||
continue;
|
||||
}
|
||||
cookieMap[name] = value;
|
||||
}
|
||||
return cookieMap;
|
||||
}
|
||||
|
||||
export function hasRequiredXSessionCookies(cookieMap: Record<string, string>): boolean {
|
||||
return REQUIRED_X_SESSION_COOKIES.every((name) => Boolean(cookieMap[name]));
|
||||
}
|
||||
|
||||
export async function readXSessionCookieMap(
|
||||
context: Pick<AdapterContext, "browser">,
|
||||
): Promise<Record<string, string>> {
|
||||
const { cookies } = await context.browser.targetSession.send<NetworkGetCookiesResult>(
|
||||
"Network.getCookies",
|
||||
{ urls: [...X_SESSION_URLS] },
|
||||
);
|
||||
return buildXSessionCookieMap(cookies ?? []);
|
||||
}
|
||||
|
||||
export async function isXSessionReady(
|
||||
context: Pick<AdapterContext, "browser">,
|
||||
): Promise<boolean> {
|
||||
const cookieMap = await readXSessionCookieMap(context);
|
||||
return hasRequiredXSessionCookies(cookieMap);
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
import path from "node:path";
|
||||
import type { NetworkEntry } from "../../browser/network-journal";
|
||||
import type { XMedia, XQuotedTweet, XTweet, XUser, JsonObject } from "./types";
|
||||
|
||||
const X_IMAGE_EXTENSIONS = new Set(["jpg", "jpeg", "png", "webp", "gif", "bmp", "avif"]);
|
||||
|
||||
function emptyObject(): JsonObject {
|
||||
return {};
|
||||
}
|
||||
|
||||
export function isRecord(value: unknown): value is JsonObject {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function walk(value: unknown, visitor: (node: unknown) => boolean | void): boolean {
|
||||
if (visitor(value)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
if (walk(item, visitor)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRecord(value)) {
|
||||
for (const child of Object.values(value)) {
|
||||
if (walk(child, visitor)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasTweetText(node: JsonObject): boolean {
|
||||
const legacy = isRecord(node.legacy) ? node.legacy : emptyObject();
|
||||
return (
|
||||
typeof legacy.full_text === "string" ||
|
||||
typeof getNoteTweetText(node) === "string"
|
||||
);
|
||||
}
|
||||
|
||||
export function findTweetNodeById(payload: unknown, tweetId: string): JsonObject | null {
|
||||
let match: JsonObject | null = null;
|
||||
|
||||
walk(payload, (node) => {
|
||||
if (!isRecord(node) || typeof node.rest_id !== "string" || !isRecord(node.legacy)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!hasTweetText(node)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (node.rest_id === tweetId) {
|
||||
match = node;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
|
||||
return match;
|
||||
}
|
||||
|
||||
export function findTweetNode(payload: unknown, statusId: string): JsonObject | null {
|
||||
let firstMatch: JsonObject | null = null;
|
||||
const exactMatch = findTweetNodeById(payload, statusId);
|
||||
if (exactMatch) {
|
||||
return exactMatch;
|
||||
}
|
||||
|
||||
walk(payload, (node) => {
|
||||
if (!isRecord(node) || typeof node.rest_id !== "string" || !isRecord(node.legacy)) {
|
||||
return false;
|
||||
}
|
||||
if (!hasTweetText(node)) {
|
||||
return false;
|
||||
}
|
||||
if (!firstMatch) {
|
||||
firstMatch = node;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
return firstMatch;
|
||||
}
|
||||
|
||||
export function getLegacy(tweet: JsonObject): JsonObject {
|
||||
return isRecord(tweet.legacy) ? tweet.legacy : emptyObject();
|
||||
}
|
||||
|
||||
export function unwrapTweetResult(node: unknown): JsonObject | null {
|
||||
if (!isRecord(node)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (node.__typename === "TweetWithVisibilityResults" && isRecord(node.tweet)) {
|
||||
return unwrapTweetResult(node.tweet);
|
||||
}
|
||||
|
||||
const tweet = isRecord(node.tweet) ? (node.tweet as JsonObject) : node;
|
||||
if (typeof tweet.rest_id !== "string" || !isRecord(tweet.legacy)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return tweet;
|
||||
}
|
||||
|
||||
export function getUser(tweet: JsonObject): XUser {
|
||||
const result =
|
||||
isRecord(tweet.core) &&
|
||||
isRecord(tweet.core.user_results) &&
|
||||
isRecord(tweet.core.user_results.result)
|
||||
? (tweet.core.user_results.result as JsonObject)
|
||||
: emptyObject();
|
||||
const legacy = isRecord(result.legacy) ? result.legacy : emptyObject();
|
||||
const core = isRecord(result.core) ? result.core : emptyObject();
|
||||
return {
|
||||
name:
|
||||
(typeof legacy.name === "string" ? legacy.name : undefined) ??
|
||||
(typeof core.name === "string" ? core.name : undefined),
|
||||
screenName:
|
||||
(typeof legacy.screen_name === "string" ? legacy.screen_name : undefined) ??
|
||||
(typeof core.screen_name === "string" ? core.screen_name : undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function getNoteTweetResult(tweet: JsonObject): JsonObject | null {
|
||||
if (
|
||||
!isRecord(tweet.note_tweet) ||
|
||||
!isRecord(tweet.note_tweet.note_tweet_results) ||
|
||||
!isRecord(tweet.note_tweet.note_tweet_results.result)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return tweet.note_tweet.note_tweet_results.result as JsonObject;
|
||||
}
|
||||
|
||||
function getNoteTweetText(tweet: JsonObject): string | undefined {
|
||||
const noteTweet = getNoteTweetResult(tweet);
|
||||
return typeof noteTweet?.text === "string" ? noteTweet.text : undefined;
|
||||
}
|
||||
|
||||
interface TweetUrlEntity {
|
||||
url: string;
|
||||
expandedUrl?: string;
|
||||
displayUrl?: string;
|
||||
}
|
||||
|
||||
function collectTweetUrlEntities(values: unknown[]): TweetUrlEntity[] {
|
||||
return values.reduce<TweetUrlEntity[]>((entities, value) => {
|
||||
if (!isRecord(value) || typeof value.url !== "string" || !value.url) {
|
||||
return entities;
|
||||
}
|
||||
|
||||
entities.push({
|
||||
url: value.url,
|
||||
expandedUrl: typeof value.expanded_url === "string" ? value.expanded_url : undefined,
|
||||
displayUrl: typeof value.display_url === "string" ? value.display_url : undefined,
|
||||
});
|
||||
|
||||
return entities;
|
||||
}, []);
|
||||
}
|
||||
|
||||
function getTweetUrlEntities(tweet: JsonObject): TweetUrlEntity[] {
|
||||
const noteTweet = getNoteTweetResult(tweet);
|
||||
const noteTweetEntitySet = noteTweet && isRecord(noteTweet.entity_set) ? noteTweet.entity_set : emptyObject();
|
||||
const noteTweetUrls = collectTweetUrlEntities(Array.isArray(noteTweetEntitySet.urls) ? noteTweetEntitySet.urls : []);
|
||||
|
||||
const legacy = getLegacy(tweet);
|
||||
const legacyEntities = isRecord(legacy.entities) ? legacy.entities : emptyObject();
|
||||
const legacyUrls = collectTweetUrlEntities(Array.isArray(legacyEntities.urls) ? legacyEntities.urls : []);
|
||||
|
||||
const seen = new Set<string>();
|
||||
return [...noteTweetUrls, ...legacyUrls].filter((value) => {
|
||||
if (seen.has(value.url)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(value.url);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function getTweetText(tweet: JsonObject): string {
|
||||
const legacy = getLegacy(tweet);
|
||||
let text =
|
||||
getNoteTweetText(tweet) ?? (typeof legacy.full_text === "string" ? legacy.full_text : "");
|
||||
|
||||
for (const value of getTweetUrlEntities(tweet)) {
|
||||
const replacement =
|
||||
(typeof value.expandedUrl === "string" && value.expandedUrl) ||
|
||||
(typeof value.displayUrl === "string" && value.displayUrl) ||
|
||||
value.url;
|
||||
text = text.replaceAll(value.url, replacement);
|
||||
}
|
||||
|
||||
const extendedEntities = isRecord(legacy.extended_entities) ? legacy.extended_entities : emptyObject();
|
||||
const media = Array.isArray(extendedEntities.media) ? extendedEntities.media : [];
|
||||
for (const value of media) {
|
||||
if (isRecord(value) && typeof value.url === "string") {
|
||||
text = text.replaceAll(value.url, "").trim();
|
||||
}
|
||||
}
|
||||
|
||||
return text.replace(/\n{3,}/g, "\n\n").trim();
|
||||
}
|
||||
|
||||
function normalizeXImageExtension(raw: string | undefined | null): string | undefined {
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const normalized = raw.replace(/^\./, "").trim().toLowerCase();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return normalized === "jpeg" ? "jpg" : normalized;
|
||||
}
|
||||
|
||||
export function toHighResXImageUrl(rawUrl: string): string {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.hostname.toLowerCase() !== "pbs.twimg.com") {
|
||||
return rawUrl;
|
||||
}
|
||||
|
||||
const pathExtension = normalizeXImageExtension(path.posix.extname(parsed.pathname));
|
||||
const format = normalizeXImageExtension(parsed.searchParams.get("format")) ?? pathExtension;
|
||||
if (!format || !X_IMAGE_EXTENSIONS.has(format)) {
|
||||
return rawUrl;
|
||||
}
|
||||
|
||||
if (pathExtension) {
|
||||
parsed.pathname = parsed.pathname.replace(new RegExp(`\\.${pathExtension}$`, "i"), "");
|
||||
}
|
||||
|
||||
parsed.searchParams.set("format", format);
|
||||
parsed.searchParams.set("name", "4096x4096");
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function getVideoVariantBitrate(variant: JsonObject): number {
|
||||
const value = variant.bitrate ?? variant.bit_rate;
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function getVideoVariantContentType(variant: JsonObject): string {
|
||||
const value = variant.content_type ?? variant.contentType;
|
||||
return typeof value === "string" ? value.toLowerCase() : "";
|
||||
}
|
||||
|
||||
export function resolveBestXVideoVariantUrl(mediaInfo: unknown): string | undefined {
|
||||
if (!isRecord(mediaInfo)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const variantsSource =
|
||||
Array.isArray(mediaInfo.variants)
|
||||
? mediaInfo.variants
|
||||
: isRecord(mediaInfo.video_info) && Array.isArray(mediaInfo.video_info.variants)
|
||||
? mediaInfo.video_info.variants
|
||||
: [];
|
||||
|
||||
const variants = variantsSource
|
||||
.filter(
|
||||
(variant): variant is JsonObject =>
|
||||
isRecord(variant) && typeof variant.url === "string" && variant.url.length > 0,
|
||||
)
|
||||
.filter((variant) => getVideoVariantContentType(variant) === "video/mp4")
|
||||
.sort((left, right) => getVideoVariantBitrate(right) - getVideoVariantBitrate(left));
|
||||
|
||||
return typeof variants[0]?.url === "string" ? variants[0].url : undefined;
|
||||
}
|
||||
|
||||
export function getTweetMedia(tweet: JsonObject): XMedia[] {
|
||||
const legacy = getLegacy(tweet);
|
||||
const extendedEntities = isRecord(legacy.extended_entities) ? legacy.extended_entities : emptyObject();
|
||||
const media = Array.isArray(extendedEntities.media) ? extendedEntities.media : [];
|
||||
|
||||
return media
|
||||
.map((value) => {
|
||||
if (!isRecord(value) || typeof value.type !== "string") {
|
||||
return null;
|
||||
}
|
||||
if (value.type === "photo" && typeof value.media_url_https === "string") {
|
||||
return {
|
||||
type: value.type,
|
||||
url: toHighResXImageUrl(value.media_url_https),
|
||||
alt: typeof value.ext_alt_text === "string" ? value.ext_alt_text : undefined,
|
||||
};
|
||||
}
|
||||
if (value.type === "video" || value.type === "animated_gif") {
|
||||
const videoUrl = resolveBestXVideoVariantUrl(value);
|
||||
if (!videoUrl) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: value.type,
|
||||
url: videoUrl,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter((value): value is XMedia => value !== null);
|
||||
}
|
||||
|
||||
export function getTweetUrl(tweet: JsonObject, fallbackUrl: string): string {
|
||||
const user = getUser(tweet);
|
||||
const fallbackScreenName = extractScreenNameFromUrl(fallbackUrl);
|
||||
const id = typeof tweet.rest_id === "string" ? tweet.rest_id : "";
|
||||
const screenName = user.screenName ?? fallbackScreenName;
|
||||
if (screenName && id) {
|
||||
return `https://x.com/${screenName}/status/${id}`;
|
||||
}
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
export function getQuotedTweet(tweet: JsonObject, fallbackUrl: string): XQuotedTweet | undefined {
|
||||
const quoted = unwrapTweetResult(
|
||||
isRecord(tweet.quoted_status_result) ? tweet.quoted_status_result.result : null,
|
||||
);
|
||||
if (!quoted) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const user = getUser(quoted);
|
||||
return {
|
||||
id: typeof quoted.rest_id === "string" ? quoted.rest_id : "",
|
||||
author: user.screenName,
|
||||
authorName: user.name,
|
||||
text: getTweetText(quoted),
|
||||
url: getTweetUrl(quoted, fallbackUrl),
|
||||
media: getTweetMedia(quoted),
|
||||
};
|
||||
}
|
||||
|
||||
export function extractScreenNameFromUrl(url: string): string | undefined {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const match = parsed.pathname.match(/^\/([^/]+)\/(?:status|article)\//);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
if (match[1] === "i") {
|
||||
return undefined;
|
||||
}
|
||||
return match[1];
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function toXTweet(tweet: JsonObject, fallbackUrl: string): XTweet {
|
||||
const legacy = getLegacy(tweet);
|
||||
const user = getUser(tweet);
|
||||
const fallbackScreenName = extractScreenNameFromUrl(fallbackUrl);
|
||||
const screenName = user.screenName ?? fallbackScreenName;
|
||||
return {
|
||||
id: typeof tweet.rest_id === "string" ? tweet.rest_id : "",
|
||||
author: screenName,
|
||||
authorName: user.name,
|
||||
text: getTweetText(tweet),
|
||||
likes: typeof legacy.favorite_count === "number" ? legacy.favorite_count : 0,
|
||||
retweets: typeof legacy.retweet_count === "number" ? legacy.retweet_count : 0,
|
||||
replies: typeof legacy.reply_count === "number" ? legacy.reply_count : 0,
|
||||
createdAt: typeof legacy.created_at === "string" ? legacy.created_at : undefined,
|
||||
inReplyTo: typeof legacy.in_reply_to_status_id_str === "string" ? legacy.in_reply_to_status_id_str : undefined,
|
||||
url: getTweetUrl(tweet, fallbackUrl),
|
||||
media: getTweetMedia(tweet),
|
||||
quotedTweet: getQuotedTweet(tweet, fallbackUrl),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeTitle(text: string, fallback: string): string {
|
||||
const firstLine = text.split("\n")[0]?.trim();
|
||||
if (!firstLine) {
|
||||
return fallback;
|
||||
}
|
||||
return firstLine.slice(0, 120);
|
||||
}
|
||||
|
||||
export function formatTweetAuthor(tweet: XTweet): string | undefined {
|
||||
if (tweet.author && tweet.authorName) {
|
||||
return `${tweet.authorName} (@${tweet.author})`;
|
||||
}
|
||||
if (tweet.author) {
|
||||
return `@${tweet.author}`;
|
||||
}
|
||||
return tweet.authorName;
|
||||
}
|
||||
|
||||
export function getTweetAuthorMetadata(tweet: XTweet): Record<string, unknown> {
|
||||
return {
|
||||
authorName: tweet.authorName,
|
||||
authorUsername: tweet.author,
|
||||
authorUrl: tweet.author ? `https://x.com/${tweet.author}` : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatMediaList(media: XMedia[]): string[] {
|
||||
return media.map((item) => {
|
||||
if (item.type === "photo") {
|
||||
return `photo: ${item.url}`;
|
||||
}
|
||||
return `${item.type}: ${item.url}`;
|
||||
});
|
||||
}
|
||||
|
||||
export function filterXGraphQlEntries(entries: NetworkEntry[]): NetworkEntry[] {
|
||||
return entries.filter((entry) => entry.url.includes("/graphql/"));
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import type { ExtractedDocument, ContentBlock } from "../../extract/document";
|
||||
import { findTweetNode, formatMediaList, formatTweetAuthor, getTweetAuthorMetadata, normalizeTitle, toXTweet } from "./shared";
|
||||
|
||||
export function extractSingleTweetDocumentFromPayload(
|
||||
payload: unknown,
|
||||
statusId: string,
|
||||
pageUrl: string,
|
||||
): ExtractedDocument | null {
|
||||
const tweet = findTweetNode(payload, statusId);
|
||||
if (!tweet) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const xTweet = toXTweet(tweet, pageUrl);
|
||||
const content: ContentBlock[] = [];
|
||||
|
||||
if (xTweet.text) {
|
||||
content.push({ type: "paragraph", text: xTweet.text });
|
||||
}
|
||||
|
||||
for (const mediaLine of formatMediaList(xTweet.media)) {
|
||||
if (mediaLine.startsWith("photo: ")) {
|
||||
content.push({
|
||||
type: "image",
|
||||
url: mediaLine.slice("photo: ".length),
|
||||
});
|
||||
} else {
|
||||
content.push({
|
||||
type: "list",
|
||||
ordered: false,
|
||||
items: [mediaLine],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (xTweet.quotedTweet) {
|
||||
const quotedLines: string[] = [];
|
||||
const quotedAuthor =
|
||||
xTweet.quotedTweet.author && xTweet.quotedTweet.authorName
|
||||
? `${xTweet.quotedTweet.authorName} (@${xTweet.quotedTweet.author})`
|
||||
: xTweet.quotedTweet.author
|
||||
? `@${xTweet.quotedTweet.author}`
|
||||
: xTweet.quotedTweet.authorName;
|
||||
|
||||
if (quotedAuthor) {
|
||||
quotedLines.push(quotedAuthor);
|
||||
}
|
||||
if (xTweet.quotedTweet.text) {
|
||||
quotedLines.push(xTweet.quotedTweet.text);
|
||||
}
|
||||
quotedLines.push(...formatMediaList(xTweet.quotedTweet.media));
|
||||
|
||||
if (quotedLines.length > 0) {
|
||||
content.push({ type: "heading", depth: 2, text: "Quoted Tweet" });
|
||||
content.push({ type: "quote", text: quotedLines.join("\n\n") });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
canonicalUrl: xTweet.url,
|
||||
title: normalizeTitle(
|
||||
xTweet.author ? `@${xTweet.author}: ${xTweet.text}` : xTweet.text,
|
||||
"Tweet",
|
||||
),
|
||||
author: formatTweetAuthor(xTweet),
|
||||
siteName: "X",
|
||||
publishedAt: xTweet.createdAt,
|
||||
summary: xTweet.text.slice(0, 200) || undefined,
|
||||
adapter: "x",
|
||||
metadata: {
|
||||
kind: "x/post",
|
||||
tweetId: xTweet.id,
|
||||
...getTweetAuthorMetadata(xTweet),
|
||||
conversationId:
|
||||
typeof tweet.legacy === "object" &&
|
||||
tweet.legacy !== null &&
|
||||
typeof (tweet.legacy as Record<string, unknown>).conversation_id_str === "string"
|
||||
? (tweet.legacy as Record<string, unknown>).conversation_id_str
|
||||
: undefined,
|
||||
favoriteCount: xTweet.likes,
|
||||
replyCount: xTweet.replies,
|
||||
retweetCount: xTweet.retweets,
|
||||
},
|
||||
content,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
import type { AdapterContext } from "../types";
|
||||
import { extractThreadTweetsFromPayloads } from "./thread";
|
||||
import { collectXJsonPayloads, getRelevantXThreadEntries, prefetchRelevantXThreadBodies } from "./payloads";
|
||||
|
||||
interface ClickTextResult {
|
||||
clicked: boolean;
|
||||
text?: string;
|
||||
}
|
||||
|
||||
interface ScrollStepResult {
|
||||
moved: boolean;
|
||||
atTop: boolean;
|
||||
atBottom: boolean;
|
||||
}
|
||||
|
||||
interface ThreadProgress {
|
||||
tweetCount: number;
|
||||
firstTweetId?: string;
|
||||
lastTweetId?: string;
|
||||
requestCount: number;
|
||||
tweetDetailCount: number;
|
||||
}
|
||||
|
||||
interface TopProbeState {
|
||||
requestCount: number;
|
||||
tweetDetailCount: number;
|
||||
scrollHeight: number;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function waitForXNetworkSettle(context: AdapterContext, reason: string): Promise<void> {
|
||||
try {
|
||||
await context.network.waitForIdle({
|
||||
idleMs: 650,
|
||||
timeoutMs: Math.min(context.timeoutMs, 5_000),
|
||||
});
|
||||
} catch {
|
||||
context.log.debug(`Network idle timed out after ${reason}.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureTopProbeState(context: AdapterContext): Promise<TopProbeState> {
|
||||
const entries = getRelevantXThreadEntries(context);
|
||||
const scrollHeight = await context.browser.evaluate<number>(`
|
||||
(() => {
|
||||
const scrollRoot = document.scrollingElement ?? document.documentElement ?? document.body;
|
||||
return scrollRoot.scrollHeight;
|
||||
})()
|
||||
`);
|
||||
|
||||
return {
|
||||
requestCount: entries.length,
|
||||
tweetDetailCount: entries.filter((entry) => entry.url.includes("TweetDetail")).length,
|
||||
scrollHeight,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForTopProbe(context: AdapterContext): Promise<boolean> {
|
||||
const initial = await captureTopProbeState(context);
|
||||
const deadline = Date.now() + 1_200;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
await context.network.waitForIdle({
|
||||
idleMs: 250,
|
||||
timeoutMs: 350,
|
||||
});
|
||||
} catch {
|
||||
// Keep polling until the shorter top-probe budget expires.
|
||||
}
|
||||
|
||||
await prefetchRelevantXThreadBodies(context);
|
||||
const next = await captureTopProbeState(context);
|
||||
if (
|
||||
next.requestCount > initial.requestCount ||
|
||||
next.tweetDetailCount > initial.tweetDetailCount ||
|
||||
next.scrollHeight > initial.scrollHeight + 4
|
||||
) {
|
||||
context.log.debug("Observed additional X thread activity while probing the page top.");
|
||||
return true;
|
||||
}
|
||||
|
||||
await sleep(120);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function scrollThreadToTop(context: AdapterContext): Promise<void> {
|
||||
let settledTopChecks = 0;
|
||||
|
||||
while (settledTopChecks < 2) {
|
||||
const scroll = await context.browser.evaluate<ScrollStepResult>(`
|
||||
(() => {
|
||||
const scrollRoot = document.scrollingElement ?? document.documentElement ?? document.body;
|
||||
const before = window.scrollY;
|
||||
window.scrollTo({ top: 0, left: 0, behavior: "instant" });
|
||||
const after = window.scrollY;
|
||||
return {
|
||||
moved: after !== before,
|
||||
atTop: after <= 4,
|
||||
atBottom: window.innerHeight + after >= scrollRoot.scrollHeight - 4,
|
||||
};
|
||||
})()
|
||||
`);
|
||||
await sleep(140);
|
||||
await waitForXNetworkSettle(context, "scrolling X thread to top");
|
||||
await prefetchRelevantXThreadBodies(context);
|
||||
|
||||
if (scroll.moved) {
|
||||
settledTopChecks = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
const observedTopActivity = await waitForTopProbe(context);
|
||||
if (observedTopActivity) {
|
||||
settledTopChecks = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
settledTopChecks += 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function clickVisibleShowReplies(context: AdapterContext): Promise<ClickTextResult> {
|
||||
return context.browser.evaluate<ClickTextResult>(`
|
||||
(() => {
|
||||
const normalize = (value) => value.replace(/\\s+/g, " ").trim();
|
||||
const matches = [
|
||||
/^Show replies$/i,
|
||||
/^Show more replies$/i,
|
||||
/^Show additional replies$/i,
|
||||
/^显示回复$/,
|
||||
/^展开回复$/,
|
||||
];
|
||||
const isVisible = (element) => {
|
||||
if (!(element instanceof HTMLElement)) {
|
||||
return false;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(element);
|
||||
return (
|
||||
rect.width > 0 &&
|
||||
rect.height > 0 &&
|
||||
style.visibility !== "hidden" &&
|
||||
style.display !== "none"
|
||||
);
|
||||
};
|
||||
|
||||
const selectors = [
|
||||
"a",
|
||||
"button",
|
||||
'[role="button"]',
|
||||
'[role="link"]',
|
||||
];
|
||||
|
||||
for (const element of document.querySelectorAll(selectors.join(","))) {
|
||||
if (!isVisible(element)) {
|
||||
continue;
|
||||
}
|
||||
const text = normalize(element.textContent ?? "");
|
||||
if (!text || !matches.some((pattern) => pattern.test(text))) {
|
||||
continue;
|
||||
}
|
||||
element.scrollIntoView({ block: "center", inline: "nearest" });
|
||||
if (element instanceof HTMLElement) {
|
||||
element.click();
|
||||
return { clicked: true, text };
|
||||
}
|
||||
}
|
||||
|
||||
return { clicked: false };
|
||||
})()
|
||||
`);
|
||||
}
|
||||
|
||||
async function expandVisibleShowReplies(context: AdapterContext): Promise<number> {
|
||||
let clickCount = 0;
|
||||
|
||||
while (clickCount < 8) {
|
||||
const result = await clickVisibleShowReplies(context).catch<ClickTextResult>(() => ({ clicked: false }));
|
||||
if (!result.clicked) {
|
||||
break;
|
||||
}
|
||||
|
||||
clickCount += 1;
|
||||
context.log.debug(`Expanded X thread replies via "${result.text ?? "Show replies"}".`);
|
||||
await sleep(250);
|
||||
await waitForXNetworkSettle(context, "expanding Show replies");
|
||||
await prefetchRelevantXThreadBodies(context);
|
||||
}
|
||||
|
||||
return clickCount;
|
||||
}
|
||||
|
||||
async function scrollThreadBy(context: AdapterContext, stepPx: number): Promise<ScrollStepResult> {
|
||||
const result = await context.browser.evaluate<ScrollStepResult>(`
|
||||
(() => {
|
||||
const scrollRoot = document.scrollingElement ?? document.documentElement ?? document.body;
|
||||
const before = window.scrollY;
|
||||
window.scrollBy({ top: ${stepPx}, left: 0, behavior: "instant" });
|
||||
const after = window.scrollY;
|
||||
return {
|
||||
moved: after !== before,
|
||||
atTop: after <= 4,
|
||||
atBottom: window.innerHeight + after >= scrollRoot.scrollHeight - 4,
|
||||
};
|
||||
})()
|
||||
`);
|
||||
|
||||
await sleep(140);
|
||||
await waitForXNetworkSettle(context, "scrolling X thread");
|
||||
await prefetchRelevantXThreadBodies(context);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function captureThreadProgress(context: AdapterContext, statusId: string): Promise<ThreadProgress> {
|
||||
const entries = getRelevantXThreadEntries(context);
|
||||
const payloads = await collectXJsonPayloads(context);
|
||||
const tweets = extractThreadTweetsFromPayloads(payloads, statusId, context.input.url.toString());
|
||||
return {
|
||||
tweetCount: tweets.length,
|
||||
firstTweetId: tweets[0]?.id,
|
||||
lastTweetId: tweets[tweets.length - 1]?.id,
|
||||
requestCount: entries.length,
|
||||
tweetDetailCount: entries.filter((entry) => entry.url.includes("TweetDetail")).length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadFullXThread(context: AdapterContext, statusId: string): Promise<void> {
|
||||
await scrollThreadToTop(context);
|
||||
|
||||
let progress = await captureThreadProgress(context, statusId);
|
||||
let stagnantRounds = 0;
|
||||
let roundsWithoutMovement = 0;
|
||||
let distanceWithoutThreadActivityPx = 0;
|
||||
|
||||
for (let round = 0; ; round += 1) {
|
||||
const stepPx = round < 12 ? 1_200 : 1_600;
|
||||
let expandedCount = await expandVisibleShowReplies(context);
|
||||
const scroll = await scrollThreadBy(context, stepPx);
|
||||
expandedCount += await expandVisibleShowReplies(context);
|
||||
const nextProgress = await captureThreadProgress(context, statusId);
|
||||
const grew =
|
||||
nextProgress.tweetCount > progress.tweetCount ||
|
||||
nextProgress.firstTweetId !== progress.firstTweetId ||
|
||||
nextProgress.lastTweetId !== progress.lastTweetId ||
|
||||
nextProgress.requestCount > progress.requestCount ||
|
||||
nextProgress.tweetDetailCount > progress.tweetDetailCount;
|
||||
|
||||
if (grew) {
|
||||
context.log.debug(
|
||||
`X thread progress: ${nextProgress.tweetCount} tweets (${nextProgress.firstTweetId ?? "unknown"} -> ${nextProgress.lastTweetId ?? "unknown"}), ${nextProgress.requestCount} requests, ${nextProgress.tweetDetailCount} TweetDetail.`,
|
||||
);
|
||||
stagnantRounds = 0;
|
||||
distanceWithoutThreadActivityPx = 0;
|
||||
} else if (expandedCount > 0) {
|
||||
stagnantRounds = 0;
|
||||
distanceWithoutThreadActivityPx = 0;
|
||||
} else {
|
||||
stagnantRounds += 1;
|
||||
distanceWithoutThreadActivityPx += stepPx;
|
||||
}
|
||||
|
||||
roundsWithoutMovement = scroll.moved ? 0 : roundsWithoutMovement + 1;
|
||||
progress = nextProgress;
|
||||
|
||||
if (scroll.atBottom && stagnantRounds >= 6) {
|
||||
context.log.debug("Stopping X thread scroll after reaching page bottom with no further thread progress.");
|
||||
break;
|
||||
}
|
||||
|
||||
if (roundsWithoutMovement >= 2 && stagnantRounds >= 4) {
|
||||
context.log.debug("Stopping X thread scroll after repeated downward scrolls no longer move the page.");
|
||||
break;
|
||||
}
|
||||
|
||||
if (distanceWithoutThreadActivityPx >= 24_000 && stagnantRounds >= 12) {
|
||||
context.log.debug("Stopping X thread scroll after a long stretch with no thread-related progress.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
import type { ExtractedDocument } from "../../extract/document";
|
||||
import {
|
||||
formatMediaList,
|
||||
formatTweetAuthor,
|
||||
getLegacy,
|
||||
getTweetAuthorMetadata,
|
||||
isRecord,
|
||||
normalizeTitle,
|
||||
toXTweet,
|
||||
unwrapTweetResult,
|
||||
} from "./shared";
|
||||
import type { JsonObject, XQuotedTweet, XTweet } from "./types";
|
||||
|
||||
interface ParsedThreadTweet extends XTweet {
|
||||
userId?: string;
|
||||
conversationId?: string;
|
||||
inReplyToUserId?: string;
|
||||
sortTimestamp: number;
|
||||
}
|
||||
|
||||
function compareTweetIds(left: string, right: string): number {
|
||||
try {
|
||||
const leftId = BigInt(left);
|
||||
const rightId = BigInt(right);
|
||||
if (leftId === rightId) {
|
||||
return 0;
|
||||
}
|
||||
return leftId < rightId ? -1 : 1;
|
||||
} catch {
|
||||
return left.localeCompare(right);
|
||||
}
|
||||
}
|
||||
|
||||
function toTimestamp(value: string | undefined): number {
|
||||
if (!value) {
|
||||
return 0;
|
||||
}
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? 0 : parsed;
|
||||
}
|
||||
|
||||
function scoreParsedTweet(tweet: ParsedThreadTweet): number {
|
||||
return (
|
||||
(tweet.text ? 4 : 0) +
|
||||
(tweet.author ? 2 : 0) +
|
||||
(tweet.authorName ? 2 : 0) +
|
||||
(tweet.media.length > 0 ? 1 : 0)
|
||||
);
|
||||
}
|
||||
|
||||
function toParsedThreadTweet(tweet: JsonObject, pageUrl: string): ParsedThreadTweet {
|
||||
const legacy = getLegacy(tweet);
|
||||
const xTweet = toXTweet(tweet, pageUrl);
|
||||
|
||||
return {
|
||||
...xTweet,
|
||||
userId: typeof legacy.user_id_str === "string" ? legacy.user_id_str : undefined,
|
||||
conversationId: typeof legacy.conversation_id_str === "string" ? legacy.conversation_id_str : undefined,
|
||||
inReplyToUserId: typeof legacy.in_reply_to_user_id_str === "string" ? legacy.in_reply_to_user_id_str : undefined,
|
||||
sortTimestamp: toTimestamp(xTweet.createdAt),
|
||||
};
|
||||
}
|
||||
|
||||
function collectTweetFromItemContent(
|
||||
itemContent: unknown,
|
||||
pageUrl: string,
|
||||
tweets: Map<string, ParsedThreadTweet>,
|
||||
): void {
|
||||
if (!isRecord(itemContent)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const tweet = unwrapTweetResult(
|
||||
isRecord(itemContent.tweet_results) ? itemContent.tweet_results.result : null,
|
||||
);
|
||||
if (!tweet || typeof tweet.rest_id !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
const parsed = toParsedThreadTweet(tweet, pageUrl);
|
||||
const existing = tweets.get(parsed.id);
|
||||
if (!existing || scoreParsedTweet(parsed) >= scoreParsedTweet(existing)) {
|
||||
tweets.set(parsed.id, parsed);
|
||||
}
|
||||
}
|
||||
|
||||
function collectTweetsFromItems(
|
||||
items: unknown,
|
||||
pageUrl: string,
|
||||
tweets: Map<string, ParsedThreadTweet>,
|
||||
): void {
|
||||
if (!Array.isArray(items)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const item of items) {
|
||||
if (!isRecord(item)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isRecord(item.item) && isRecord(item.item.itemContent)) {
|
||||
collectTweetFromItemContent(item.item.itemContent, pageUrl, tweets);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isRecord(item.itemContent)) {
|
||||
collectTweetFromItemContent(item.itemContent, pageUrl, tweets);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getInstructions(payload: unknown): unknown[] {
|
||||
if (!isRecord(payload) || !isRecord(payload.data)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { data } = payload;
|
||||
return (
|
||||
(isRecord(data.threaded_conversation_with_injections_v2) &&
|
||||
Array.isArray(data.threaded_conversation_with_injections_v2.instructions)
|
||||
? data.threaded_conversation_with_injections_v2.instructions
|
||||
: undefined) ??
|
||||
(isRecord(data.threaded_conversation_with_injections) &&
|
||||
Array.isArray(data.threaded_conversation_with_injections.instructions)
|
||||
? data.threaded_conversation_with_injections.instructions
|
||||
: undefined) ??
|
||||
(isRecord(data.tweetResult) &&
|
||||
isRecord(data.tweetResult.result) &&
|
||||
isRecord(data.tweetResult.result.timeline) &&
|
||||
Array.isArray(data.tweetResult.result.timeline.instructions)
|
||||
? data.tweetResult.result.timeline.instructions
|
||||
: [])
|
||||
);
|
||||
}
|
||||
|
||||
function parseTweetDetailPayload(payload: unknown, pageUrl: string): ParsedThreadTweet[] {
|
||||
const tweets = new Map<string, ParsedThreadTweet>();
|
||||
|
||||
const instructions = getInstructions(payload);
|
||||
for (const instruction of instructions) {
|
||||
if (!isRecord(instruction)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
collectTweetsFromItems(instruction.moduleItems, pageUrl, tweets);
|
||||
|
||||
if (!Array.isArray(instruction.entries)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const entry of instruction.entries) {
|
||||
if (!isRecord(entry)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = isRecord(entry.content) ? entry.content : {};
|
||||
collectTweetFromItemContent(content.itemContent, pageUrl, tweets);
|
||||
collectTweetsFromItems(content.items, pageUrl, tweets);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(tweets.values());
|
||||
}
|
||||
|
||||
function buildContinuousThread(tweets: ParsedThreadTweet[], statusId: string): ParsedThreadTweet[] {
|
||||
const byId = new Map<string, ParsedThreadTweet>();
|
||||
for (const tweet of tweets) {
|
||||
const existing = byId.get(tweet.id);
|
||||
if (!existing || scoreParsedTweet(tweet) >= scoreParsedTweet(existing)) {
|
||||
byId.set(tweet.id, tweet);
|
||||
}
|
||||
}
|
||||
|
||||
const rootTweet = byId.get(statusId);
|
||||
if (!rootTweet?.userId || !rootTweet.conversationId) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const candidates = Array.from(byId.values()).filter(
|
||||
(tweet) =>
|
||||
tweet.id === statusId ||
|
||||
(tweet.userId === rootTweet.userId && tweet.conversationId === rootTweet.conversationId),
|
||||
);
|
||||
|
||||
const repliesByParent = new Map<string, ParsedThreadTweet[]>();
|
||||
for (const tweet of candidates) {
|
||||
if (!tweet.inReplyTo || tweet.id === statusId) {
|
||||
continue;
|
||||
}
|
||||
const bucket = repliesByParent.get(tweet.inReplyTo) ?? [];
|
||||
bucket.push(tweet);
|
||||
bucket.sort((left, right) => {
|
||||
if (left.sortTimestamp !== right.sortTimestamp) {
|
||||
return left.sortTimestamp - right.sortTimestamp;
|
||||
}
|
||||
return compareTweetIds(left.id, right.id);
|
||||
});
|
||||
repliesByParent.set(tweet.inReplyTo, bucket);
|
||||
}
|
||||
|
||||
const ancestorPath: ParsedThreadTweet[] = [rootTweet];
|
||||
const ancestorSeen = new Set<string>([rootTweet.id]);
|
||||
let currentAncestor = rootTweet;
|
||||
|
||||
while (currentAncestor.inReplyTo) {
|
||||
const parent = byId.get(currentAncestor.inReplyTo);
|
||||
if (!parent || ancestorSeen.has(parent.id)) {
|
||||
break;
|
||||
}
|
||||
ancestorPath.unshift(parent);
|
||||
ancestorSeen.add(parent.id);
|
||||
currentAncestor = parent;
|
||||
}
|
||||
|
||||
const chain = ancestorPath.slice();
|
||||
const seen = new Set<string>(chain.map((tweet) => tweet.id));
|
||||
let currentId = rootTweet.id;
|
||||
|
||||
while (true) {
|
||||
const next = (repliesByParent.get(currentId) ?? []).find((tweet) => !seen.has(tweet.id));
|
||||
if (!next) {
|
||||
break;
|
||||
}
|
||||
chain.push(next);
|
||||
seen.add(next.id);
|
||||
currentId = next.id;
|
||||
}
|
||||
|
||||
return chain;
|
||||
}
|
||||
|
||||
export function extractThreadTweetsFromPayloads(
|
||||
payloads: unknown[],
|
||||
statusId: string,
|
||||
pageUrl: string,
|
||||
): XTweet[] {
|
||||
const parsedTweets: ParsedThreadTweet[] = [];
|
||||
|
||||
for (const payload of payloads) {
|
||||
parsedTweets.push(...parseTweetDetailPayload(payload, pageUrl));
|
||||
}
|
||||
|
||||
return buildContinuousThread(parsedTweets, statusId).map(({ sortTimestamp: _sortTimestamp, ...tweet }) => tweet);
|
||||
}
|
||||
|
||||
function buildQuotedTweetMarkdown(quotedTweet: XQuotedTweet): string {
|
||||
const author = quotedTweet.author ? `@${quotedTweet.author}` : "Unknown";
|
||||
const name = quotedTweet.authorName ? `${quotedTweet.authorName} ` : "";
|
||||
const lines: string[] = [`Quoted Tweet${quotedTweet.author || quotedTweet.authorName ? `: ${name}${author}`.trim() : ""}`];
|
||||
|
||||
if (quotedTweet.text) {
|
||||
lines.push(...quotedTweet.text.split("\n"));
|
||||
}
|
||||
|
||||
for (const mediaLine of formatMediaList(quotedTweet.media)) {
|
||||
lines.push(mediaLine);
|
||||
}
|
||||
|
||||
return lines.map((line) => (line ? `> ${line}` : ">")).join("\n");
|
||||
}
|
||||
|
||||
function buildThreadMarkdown(tweets: XTweet[]): string {
|
||||
return tweets
|
||||
.map((tweet, index) => {
|
||||
const lines: string[] = [];
|
||||
const author = tweet.author ? `@${tweet.author}` : "Unknown";
|
||||
const name = tweet.authorName ? `${tweet.authorName} ` : "";
|
||||
lines.push(`## ${index + 1}. ${name}${author}`.trim());
|
||||
if (tweet.createdAt) {
|
||||
lines.push(`_Published: ${tweet.createdAt}_`);
|
||||
}
|
||||
lines.push(tweet.text || "(No text)");
|
||||
const mediaLines = formatMediaList(tweet.media);
|
||||
if (mediaLines.length > 0) {
|
||||
lines.push(mediaLines.map((line) => `- ${line}`).join("\n"));
|
||||
}
|
||||
if (tweet.quotedTweet) {
|
||||
lines.push(buildQuotedTweetMarkdown(tweet.quotedTweet));
|
||||
}
|
||||
return lines.join("\n\n");
|
||||
})
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
export function extractThreadDocumentFromPayloads(
|
||||
payloads: unknown[],
|
||||
statusId: string,
|
||||
pageUrl: string,
|
||||
): ExtractedDocument | null {
|
||||
const tweets = extractThreadTweetsFromPayloads(payloads, statusId, pageUrl);
|
||||
if (tweets.length <= 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rootTweet = tweets[0];
|
||||
const rootAuthor = formatTweetAuthor(rootTweet);
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
canonicalUrl: rootTweet.url,
|
||||
title: normalizeTitle(rootTweet.text, "X Thread"),
|
||||
author: rootAuthor,
|
||||
siteName: "X",
|
||||
publishedAt: rootTweet.createdAt,
|
||||
summary: rootTweet.text.slice(0, 200) || undefined,
|
||||
adapter: "x",
|
||||
metadata: {
|
||||
kind: "x/thread",
|
||||
tweetId: rootTweet.id,
|
||||
tweetCount: tweets.length,
|
||||
lastTweetId: tweets[tweets.length - 1]?.id,
|
||||
...getTweetAuthorMetadata(rootTweet),
|
||||
},
|
||||
content: [{ type: "markdown", markdown: buildThreadMarkdown(tweets) }],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
export type JsonObject = Record<string, unknown>;
|
||||
|
||||
export interface XUser {
|
||||
name?: string;
|
||||
screenName?: string;
|
||||
}
|
||||
|
||||
export interface XMedia {
|
||||
type: string;
|
||||
url: string;
|
||||
alt?: string;
|
||||
}
|
||||
|
||||
export interface XQuotedTweet {
|
||||
id: string;
|
||||
author?: string;
|
||||
authorName?: string;
|
||||
text: string;
|
||||
url: string;
|
||||
media: XMedia[];
|
||||
}
|
||||
|
||||
export interface XTweet {
|
||||
id: string;
|
||||
author?: string;
|
||||
authorName?: string;
|
||||
text: string;
|
||||
likes: number;
|
||||
retweets: number;
|
||||
replies: number;
|
||||
createdAt?: string;
|
||||
inReplyTo?: string;
|
||||
url: string;
|
||||
media: XMedia[];
|
||||
quotedTweet?: XQuotedTweet;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { Adapter } from "../types";
|
||||
import { collectMediaFromDocument } from "../../media/markdown-media";
|
||||
import { extractYouTubeTranscriptDocument } from "./transcript";
|
||||
import { isYouTubeHost, parseYouTubeVideoId } from "./utils";
|
||||
|
||||
export const youtubeAdapter: Adapter = {
|
||||
name: "youtube",
|
||||
match(input) {
|
||||
return isYouTubeHost(input.url.hostname);
|
||||
},
|
||||
async process(context) {
|
||||
const videoId = parseYouTubeVideoId(context.input.url);
|
||||
if (!videoId) {
|
||||
return {
|
||||
status: "no_document",
|
||||
};
|
||||
}
|
||||
|
||||
context.log.info(`Loading ${context.input.url.toString()} with youtube adapter`);
|
||||
const document = await extractYouTubeTranscriptDocument(context, videoId);
|
||||
if (!document) {
|
||||
return {
|
||||
status: "no_document",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "ok",
|
||||
document,
|
||||
media: collectMediaFromDocument(document),
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,392 @@
|
||||
import type { ExtractedDocument } from "../../extract/document";
|
||||
import { detectInteractionGate } from "../../browser/interaction-gates";
|
||||
import {
|
||||
buildYouTubeThumbnailCandidates,
|
||||
parseYouTubeDescriptionChapters,
|
||||
renderYouTubeTranscriptMarkdown,
|
||||
type YouTubeChapter,
|
||||
type YouTubeTranscriptSegment,
|
||||
} from "./utils";
|
||||
|
||||
interface CaptionInfo {
|
||||
captionUrl: string;
|
||||
language: string;
|
||||
kind: string;
|
||||
available: string[];
|
||||
title?: string;
|
||||
author?: string;
|
||||
authorUrl?: string;
|
||||
channelId?: string;
|
||||
description?: string;
|
||||
publishedAt?: string;
|
||||
viewCount?: number;
|
||||
durationSeconds?: number;
|
||||
keywords: string[];
|
||||
category?: string;
|
||||
isLiveContent?: boolean;
|
||||
coverImages: string[];
|
||||
}
|
||||
|
||||
function normalizeUrl(url: string | undefined): string | undefined {
|
||||
if (!url) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol === "http:") {
|
||||
parsed.protocol = "https:";
|
||||
}
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
function buildSummary(description: string | undefined, segments: YouTubeTranscriptSegment[]): string | undefined {
|
||||
const descriptionSummary = description
|
||||
?.replace(/\r\n/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line && !/^https?:\/\//i.test(line));
|
||||
|
||||
if (descriptionSummary) {
|
||||
return descriptionSummary.slice(0, 240);
|
||||
}
|
||||
|
||||
const transcriptSummary = segments
|
||||
.slice(0, 8)
|
||||
.map((segment) => segment.text)
|
||||
.join(" ")
|
||||
.slice(0, 240)
|
||||
.trim();
|
||||
|
||||
return transcriptSummary || undefined;
|
||||
}
|
||||
|
||||
async function canFetchThumbnail(url: string): Promise<boolean> {
|
||||
try {
|
||||
const response = await fetch(url, { method: "HEAD", redirect: "follow" });
|
||||
if (response.ok) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (response.status === 405) {
|
||||
const fallbackResponse = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: { Range: "bytes=0-0" },
|
||||
redirect: "follow",
|
||||
});
|
||||
return fallbackResponse.ok;
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function resolveBestCoverImage(videoId: string, coverImages: string[]): Promise<string | undefined> {
|
||||
const candidates = buildYouTubeThumbnailCandidates(videoId, coverImages);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (await canFetchThumbnail(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
export async function extractYouTubeTranscriptDocument(
|
||||
context: Parameters<import("../types").Adapter["process"]>[0],
|
||||
videoId: string,
|
||||
): Promise<ExtractedDocument | null> {
|
||||
const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||
await context.browser.goto(videoUrl, context.timeoutMs);
|
||||
|
||||
const interaction = await detectInteractionGate(context.browser);
|
||||
if (interaction) {
|
||||
context.log.debug(`Interaction gate detected on YouTube: ${interaction.provider}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
await context.network.waitForIdle({
|
||||
idleMs: 1_000,
|
||||
timeoutMs: Math.min(context.timeoutMs, 8_000),
|
||||
});
|
||||
} catch {
|
||||
context.log.debug("Network idle timed out on YouTube load.");
|
||||
}
|
||||
|
||||
const captionInfo = await context.browser.evaluate<CaptionInfo | { error: string }>(`
|
||||
(async () => {
|
||||
function readText(value) {
|
||||
if (!value) return undefined;
|
||||
if (typeof value === 'string') {
|
||||
const text = value.trim();
|
||||
return text || undefined;
|
||||
}
|
||||
if (typeof value.simpleText === 'string') {
|
||||
const text = value.simpleText.trim();
|
||||
return text || undefined;
|
||||
}
|
||||
if (Array.isArray(value.runs)) {
|
||||
const text = value.runs
|
||||
.map((run) => typeof run?.text === 'string' ? run.text : '')
|
||||
.join('')
|
||||
.trim();
|
||||
return text || undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value) {
|
||||
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
|
||||
return Math.floor(value);
|
||||
}
|
||||
if (typeof value !== 'string') {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = value.replace(/[^\\d]/g, '');
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = Number.parseInt(normalized, 10);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
const apiKey = window.ytcfg?.data_?.INNERTUBE_API_KEY;
|
||||
const playerResponse = window.ytInitialPlayerResponse;
|
||||
const videoDetails = playerResponse?.videoDetails || {};
|
||||
const microformat = playerResponse?.microformat?.playerMicroformatRenderer || {};
|
||||
const title =
|
||||
videoDetails.title ||
|
||||
readText(microformat.title) ||
|
||||
document.title.replace(/ - YouTube$/, '').trim();
|
||||
const author =
|
||||
videoDetails.author ||
|
||||
microformat.ownerChannelName ||
|
||||
document.querySelector('link[itemprop="name"]')?.getAttribute('content') ||
|
||||
undefined;
|
||||
const authorUrl =
|
||||
microformat.ownerProfileUrl ||
|
||||
(typeof videoDetails.channelId === 'string' && videoDetails.channelId
|
||||
? 'https://www.youtube.com/channel/' + videoDetails.channelId
|
||||
: undefined);
|
||||
const description =
|
||||
readText(microformat.description) ||
|
||||
(typeof videoDetails.shortDescription === 'string' ? videoDetails.shortDescription.trim() : undefined);
|
||||
const keywords = Array.isArray(videoDetails.keywords)
|
||||
? videoDetails.keywords.filter((keyword) => typeof keyword === 'string' && keyword.trim())
|
||||
: [];
|
||||
const thumbnails = [
|
||||
...(Array.isArray(videoDetails.thumbnail?.thumbnails) ? videoDetails.thumbnail.thumbnails : []),
|
||||
...(Array.isArray(microformat.thumbnail?.thumbnails) ? microformat.thumbnail.thumbnails : []),
|
||||
]
|
||||
.filter((thumbnail) => typeof thumbnail?.url === 'string' && thumbnail.url)
|
||||
.sort((left, right) => ((right?.width || 0) * (right?.height || 0)) - ((left?.width || 0) * (left?.height || 0)))
|
||||
.map((thumbnail) => thumbnail.url);
|
||||
|
||||
if (!apiKey) {
|
||||
return { error: 'INNERTUBE_API_KEY not found on page' };
|
||||
}
|
||||
|
||||
const response = await fetch('/youtubei/v1/player?key=' + apiKey + '&prettyPrint=false', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
context: { client: { clientName: 'ANDROID', clientVersion: '20.10.38' } },
|
||||
videoId: ${JSON.stringify(videoId)}
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return { error: 'InnerTube player API returned HTTP ' + response.status };
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const renderer = data.captions?.playerCaptionsTracklistRenderer;
|
||||
if (!renderer?.captionTracks?.length) {
|
||||
return { error: 'No captions available for this video' };
|
||||
}
|
||||
|
||||
const tracks = renderer.captionTracks;
|
||||
const track = tracks.find((item) => item.kind !== 'asr') || tracks[0];
|
||||
|
||||
return {
|
||||
captionUrl: track.baseUrl,
|
||||
language: track.languageCode,
|
||||
kind: track.kind || 'manual',
|
||||
available: tracks.map((item) => {
|
||||
const languageLabel = readText(item.name) || item.languageCode;
|
||||
return item.kind === 'asr'
|
||||
? languageLabel + ' [' + item.languageCode + ', auto]'
|
||||
: languageLabel + ' [' + item.languageCode + ']';
|
||||
}),
|
||||
title,
|
||||
author,
|
||||
authorUrl,
|
||||
channelId: typeof videoDetails.channelId === 'string' ? videoDetails.channelId : undefined,
|
||||
description,
|
||||
publishedAt:
|
||||
(typeof microformat.publishDate === 'string' && microformat.publishDate) ||
|
||||
(typeof microformat.uploadDate === 'string' && microformat.uploadDate) ||
|
||||
document.querySelector('meta[itemprop="datePublished"]')?.getAttribute('content') ||
|
||||
undefined,
|
||||
viewCount: parsePositiveInteger(videoDetails.viewCount) ?? parsePositiveInteger(microformat.viewCount),
|
||||
durationSeconds: parsePositiveInteger(videoDetails.lengthSeconds),
|
||||
keywords,
|
||||
category: typeof microformat.category === 'string' ? microformat.category : undefined,
|
||||
isLiveContent: Boolean(videoDetails.isLiveContent || microformat.isLiveContent),
|
||||
coverImages: thumbnails,
|
||||
};
|
||||
})()
|
||||
`);
|
||||
|
||||
if ("error" in captionInfo) {
|
||||
context.log.debug(`YouTube transcript unavailable: ${captionInfo.error}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const segments = await context.browser.evaluate<YouTubeTranscriptSegment[] | { error: string }>(`
|
||||
(async () => {
|
||||
const response = await fetch(${JSON.stringify(captionInfo.captionUrl)});
|
||||
const xml = await response.text();
|
||||
if (!xml) {
|
||||
return { error: 'Caption XML is empty' };
|
||||
}
|
||||
|
||||
function getAttr(tag, name) {
|
||||
const needle = name + '="';
|
||||
const index = tag.indexOf(needle);
|
||||
if (index === -1) return '';
|
||||
const valueStart = index + needle.length;
|
||||
const valueEnd = tag.indexOf('"', valueStart);
|
||||
if (valueEnd === -1) return '';
|
||||
return tag.substring(valueStart, valueEnd);
|
||||
}
|
||||
|
||||
function decodeEntities(value) {
|
||||
return value
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll(''', "'");
|
||||
}
|
||||
|
||||
const marker = xml.includes('<p t="') ? '<p ' : '<text ';
|
||||
const endMarker = marker === '<p ' ? '</p>' : '</text>';
|
||||
const results = [];
|
||||
let position = 0;
|
||||
|
||||
while (true) {
|
||||
const tagStart = xml.indexOf(marker, position);
|
||||
if (tagStart === -1) break;
|
||||
let contentStart = xml.indexOf('>', tagStart);
|
||||
if (contentStart === -1) break;
|
||||
contentStart += 1;
|
||||
const tagEnd = xml.indexOf(endMarker, contentStart);
|
||||
if (tagEnd === -1) break;
|
||||
|
||||
const attrString = xml.substring(tagStart + marker.length, contentStart - 1);
|
||||
const content = xml.substring(contentStart, tagEnd);
|
||||
const start = marker === '<p '
|
||||
? (parseFloat(getAttr(attrString, 't')) || 0) / 1000
|
||||
: (parseFloat(getAttr(attrString, 'start')) || 0);
|
||||
const duration = marker === '<p '
|
||||
? (parseFloat(getAttr(attrString, 'd')) || 0) / 1000
|
||||
: (parseFloat(getAttr(attrString, 'dur')) || 0);
|
||||
const text = decodeEntities(content.replace(/<[^>]+>/g, '')).split('\\n').join(' ').trim();
|
||||
if (text) {
|
||||
results.push({ start, end: start + duration, text });
|
||||
}
|
||||
|
||||
position = tagEnd + endMarker.length;
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
return { error: 'Parsed 0 transcript segments' };
|
||||
}
|
||||
return results;
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!Array.isArray(segments) || segments.length === 0) {
|
||||
context.log.debug("Parsed no YouTube transcript segments.");
|
||||
return null;
|
||||
}
|
||||
|
||||
const extractedChapters = await context.browser.evaluate<YouTubeChapter[]>(`
|
||||
(() => {
|
||||
const data = window.ytInitialData;
|
||||
const markers = data?.playerOverlays?.playerOverlayRenderer
|
||||
?.decoratedPlayerBarRenderer?.decoratedPlayerBarRenderer
|
||||
?.playerBar?.multiMarkersPlayerBarRenderer?.markersMap || [];
|
||||
const results = [];
|
||||
|
||||
for (const marker of markers) {
|
||||
const chapters = marker?.value?.chapters;
|
||||
if (!Array.isArray(chapters)) continue;
|
||||
for (const chapter of chapters) {
|
||||
const renderer = chapter?.chapterRenderer;
|
||||
const title = renderer?.title?.simpleText;
|
||||
const timeRangeStartMillis = renderer?.timeRangeStartMillis;
|
||||
if (title && typeof timeRangeStartMillis === 'number') {
|
||||
results.push({ title, time: Math.floor(timeRangeStartMillis / 1000) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
})()
|
||||
`).catch(() => []);
|
||||
|
||||
const descriptionChapters = parseYouTubeDescriptionChapters(captionInfo.description);
|
||||
const chapters = extractedChapters.length > 0 ? extractedChapters : descriptionChapters;
|
||||
const markdown = renderYouTubeTranscriptMarkdown({
|
||||
description: captionInfo.description,
|
||||
segments,
|
||||
chapters,
|
||||
});
|
||||
|
||||
if (!markdown) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const pageUrl = await context.browser.getURL();
|
||||
const coverImage = await resolveBestCoverImage(videoId, captionInfo.coverImages);
|
||||
const summary = buildSummary(captionInfo.description, segments);
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
canonicalUrl: pageUrl,
|
||||
title: captionInfo.title || "YouTube Transcript",
|
||||
author: captionInfo.author,
|
||||
publishedAt: captionInfo.publishedAt,
|
||||
siteName: "YouTube",
|
||||
summary,
|
||||
adapter: "youtube",
|
||||
metadata: {
|
||||
kind: "youtube/transcript",
|
||||
videoId,
|
||||
authorUrl: normalizeUrl(captionInfo.authorUrl),
|
||||
channelId: captionInfo.channelId,
|
||||
coverImage,
|
||||
description: captionInfo.description,
|
||||
durationSeconds: captionInfo.durationSeconds,
|
||||
language: captionInfo.language,
|
||||
captionKind: captionInfo.kind,
|
||||
availableLanguages: captionInfo.available,
|
||||
viewCount: captionInfo.viewCount,
|
||||
keywords: captionInfo.keywords,
|
||||
category: captionInfo.category,
|
||||
isLiveContent: captionInfo.isLiveContent,
|
||||
chapterCount: chapters.length,
|
||||
},
|
||||
content: [{ type: "markdown", markdown }],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
export interface YouTubeTranscriptSegment {
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface YouTubeChapter {
|
||||
title: string;
|
||||
time: number;
|
||||
}
|
||||
|
||||
interface RenderYouTubeTranscriptMarkdownInput {
|
||||
description?: string;
|
||||
segments: YouTubeTranscriptSegment[];
|
||||
chapters: YouTubeChapter[];
|
||||
}
|
||||
|
||||
const DESCRIPTION_CHAPTER_RE = /^((?:\d{1,2}:)?\d{1,2}:\d{2})(?:\s+[-|:]\s+|\s+)(.+)$/;
|
||||
const YOUTUBE_THUMBNAIL_VARIANTS = [
|
||||
"maxresdefault.jpg",
|
||||
"sddefault.jpg",
|
||||
"hqdefault.jpg",
|
||||
"mqdefault.jpg",
|
||||
"default.jpg",
|
||||
];
|
||||
|
||||
export function isYouTubeHost(hostname: string): boolean {
|
||||
return [
|
||||
"youtube.com",
|
||||
"www.youtube.com",
|
||||
"m.youtube.com",
|
||||
"youtu.be",
|
||||
].includes(hostname);
|
||||
}
|
||||
|
||||
export function parseYouTubeVideoId(url: URL): string | null {
|
||||
if (url.hostname === "youtu.be") {
|
||||
return url.pathname.split("/").filter(Boolean)[0] ?? null;
|
||||
}
|
||||
|
||||
if (url.pathname === "/watch") {
|
||||
return url.searchParams.get("v");
|
||||
}
|
||||
|
||||
const shortsMatch = url.pathname.match(/^\/shorts\/([^/?#]+)/);
|
||||
if (shortsMatch) {
|
||||
return shortsMatch[1];
|
||||
}
|
||||
|
||||
const liveMatch = url.pathname.match(/^\/live\/([^/?#]+)/);
|
||||
if (liveMatch) {
|
||||
return liveMatch[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseTimestampValue(raw: string): number | null {
|
||||
const parts = raw
|
||||
.split(":")
|
||||
.map((part) => Number.parseInt(part, 10))
|
||||
.filter((part) => Number.isFinite(part));
|
||||
|
||||
if (parts.length < 2 || parts.length > 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parts.some((part) => part < 0)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parts.length === 2) {
|
||||
const [minutes, seconds] = parts;
|
||||
return minutes * 60 + seconds;
|
||||
}
|
||||
|
||||
const [hours, minutes, seconds] = parts;
|
||||
return hours * 3600 + minutes * 60 + seconds;
|
||||
}
|
||||
|
||||
export function formatTimestamp(totalSeconds: number): string {
|
||||
const rounded = Math.max(0, Math.floor(totalSeconds));
|
||||
const hours = Math.floor(rounded / 3600);
|
||||
const minutes = Math.floor((rounded % 3600) / 60);
|
||||
const seconds = rounded % 60;
|
||||
|
||||
if (hours > 0) {
|
||||
return `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`;
|
||||
}
|
||||
return `${minutes}:${String(seconds).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
export function formatTimestampRange(start: number, end: number): string {
|
||||
const safeStart = Math.max(0, start);
|
||||
const safeEnd = Math.max(safeStart, end);
|
||||
return `[${formatTimestamp(safeStart)} -> ${formatTimestamp(safeEnd)}]`;
|
||||
}
|
||||
|
||||
export function normalizeYouTubeChapters(chapters: YouTubeChapter[]): YouTubeChapter[] {
|
||||
const seenTimes = new Set<number>();
|
||||
|
||||
return chapters
|
||||
.map((chapter) => ({
|
||||
title: chapter.title.trim(),
|
||||
time: Math.max(0, Math.floor(chapter.time)),
|
||||
}))
|
||||
.filter((chapter) => chapter.title)
|
||||
.sort((left, right) => left.time - right.time)
|
||||
.filter((chapter) => {
|
||||
if (seenTimes.has(chapter.time)) {
|
||||
return false;
|
||||
}
|
||||
seenTimes.add(chapter.time);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function parseYouTubeDescriptionChapters(description?: string | null): YouTubeChapter[] {
|
||||
if (!description) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const chapters: YouTubeChapter[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const rawLine of description.replace(/\r\n/g, "\n").split("\n")) {
|
||||
const line = rawLine.trim();
|
||||
if (!line) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = line.match(DESCRIPTION_CHAPTER_RE);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const time = parseTimestampValue(match[1]);
|
||||
const title = match[2]?.trim();
|
||||
if (time === null || !title) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = `${time}:${title.toLowerCase()}`;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
seen.add(key);
|
||||
chapters.push({ title, time });
|
||||
}
|
||||
|
||||
const normalized = normalizeYouTubeChapters(chapters);
|
||||
if (normalized.length >= 2) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (normalized.length === 1 && normalized[0]?.time === 0) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function renderDescriptionMarkdown(description: string): string {
|
||||
return description
|
||||
.replace(/\r\n/g, "\n")
|
||||
.trim()
|
||||
.split(/\n{2,}/)
|
||||
.map((block) => block.split("\n").map((line) => line.trimEnd()).join(" \n"))
|
||||
.join("\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function renderSegmentLine(segment: YouTubeTranscriptSegment): string {
|
||||
return `${formatTimestampRange(segment.start, segment.end)} ${segment.text}`;
|
||||
}
|
||||
|
||||
export function renderYouTubeTranscriptMarkdown({
|
||||
description,
|
||||
segments,
|
||||
chapters,
|
||||
}: RenderYouTubeTranscriptMarkdownInput): string {
|
||||
if (segments.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
const normalizedDescription = description?.trim();
|
||||
const transcriptEnd = segments.reduce((maxEnd, segment) => Math.max(maxEnd, segment.end, segment.start), 0);
|
||||
const normalizedChapters = normalizeYouTubeChapters(chapters).filter(
|
||||
(chapter) => transcriptEnd <= 0 || chapter.time < transcriptEnd,
|
||||
);
|
||||
|
||||
if (normalizedDescription) {
|
||||
parts.push("## Description");
|
||||
parts.push(renderDescriptionMarkdown(normalizedDescription));
|
||||
}
|
||||
|
||||
if (normalizedChapters.length > 0) {
|
||||
parts.push("## Chapters");
|
||||
|
||||
for (let index = 0; index < normalizedChapters.length; index += 1) {
|
||||
const chapter = normalizedChapters[index];
|
||||
const nextChapter = normalizedChapters[index + 1];
|
||||
const chapterEnd = nextChapter ? nextChapter.time : transcriptEnd;
|
||||
const chapterSegments = segments.filter(
|
||||
(segment) => segment.start >= chapter.time && segment.start < chapterEnd,
|
||||
);
|
||||
|
||||
parts.push(`### ${chapter.title} ${formatTimestampRange(chapter.time, chapterEnd)}`);
|
||||
|
||||
if (chapterSegments.length > 0) {
|
||||
parts.push(chapterSegments.map(renderSegmentLine).join("\n"));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
parts.push("## Transcript");
|
||||
parts.push(segments.map(renderSegmentLine).join("\n"));
|
||||
}
|
||||
|
||||
return parts.filter(Boolean).join("\n\n").trim();
|
||||
}
|
||||
|
||||
function normalizeThumbnailKey(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return `${parsed.origin}${parsed.pathname}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildYouTubeThumbnailCandidates(videoId: string, listedUrls: string[]): string[] {
|
||||
const candidates = [
|
||||
...YOUTUBE_THUMBNAIL_VARIANTS.map((variant) => `https://i.ytimg.com/vi/${videoId}/${variant}`),
|
||||
...listedUrls,
|
||||
];
|
||||
|
||||
const seen = new Set<string>();
|
||||
return candidates.filter((candidate) => {
|
||||
if (!candidate) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const key = normalizeThumbnailKey(candidate);
|
||||
if (seen.has(key)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import WebSocket from "ws";
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
interface CdpPendingCommand {
|
||||
resolve(value: unknown): void;
|
||||
reject(error: unknown): void;
|
||||
method: string;
|
||||
}
|
||||
|
||||
interface CdpErrorShape {
|
||||
message?: string;
|
||||
}
|
||||
|
||||
interface CdpCommandResult<T> {
|
||||
result?: T;
|
||||
error?: CdpErrorShape;
|
||||
}
|
||||
|
||||
interface CreatePageSessionOptions {
|
||||
initialUrl?: string;
|
||||
visible?: boolean;
|
||||
}
|
||||
|
||||
export class TargetSession extends EventEmitter {
|
||||
constructor(
|
||||
private readonly client: CdpClient,
|
||||
public readonly targetId: string,
|
||||
public readonly sessionId: string,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async send<T>(method: string, params: JsonObject = {}): Promise<T> {
|
||||
return this.client.sendSessionCommand<T>(this.sessionId, method, params);
|
||||
}
|
||||
|
||||
handleEvent(method: string, params: JsonObject): void {
|
||||
this.emit(method, params);
|
||||
this.emit("event", { method, params });
|
||||
}
|
||||
|
||||
async waitForEvent<T extends JsonObject>(
|
||||
method: string,
|
||||
predicate?: (params: T) => boolean,
|
||||
timeoutMs = 30_000,
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.off(method, listener);
|
||||
reject(new Error(`Timed out waiting for ${method}`));
|
||||
}, timeoutMs);
|
||||
|
||||
const listener = (params: T): void => {
|
||||
if (predicate && !predicate(params)) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(timeout);
|
||||
this.off(method, listener);
|
||||
resolve(params);
|
||||
};
|
||||
|
||||
this.on(method, listener);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class CdpClient {
|
||||
private readonly ws: WebSocket;
|
||||
private readonly pending = new Map<number, CdpPendingCommand>();
|
||||
private readonly sessions = new Map<string, TargetSession>();
|
||||
private nextId = 1;
|
||||
|
||||
private constructor(ws: WebSocket) {
|
||||
this.ws = ws;
|
||||
this.ws.on("message", (raw) => {
|
||||
this.handleMessage(raw.toString());
|
||||
});
|
||||
}
|
||||
|
||||
static async connect(browserWsUrl: string): Promise<CdpClient> {
|
||||
const ws = await new Promise<WebSocket>((resolve, reject) => {
|
||||
const socket = new WebSocket(browserWsUrl);
|
||||
socket.once("open", () => resolve(socket));
|
||||
socket.once("error", (error) => reject(error));
|
||||
});
|
||||
|
||||
return new CdpClient(ws);
|
||||
}
|
||||
|
||||
private handleMessage(rawMessage: string): void {
|
||||
const message = JSON.parse(rawMessage) as {
|
||||
id?: number;
|
||||
sessionId?: string;
|
||||
method?: string;
|
||||
params?: JsonObject;
|
||||
result?: unknown;
|
||||
error?: CdpErrorShape;
|
||||
};
|
||||
|
||||
if (typeof message.id === "number") {
|
||||
const pending = this.pending.get(message.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
this.pending.delete(message.id);
|
||||
if (message.error) {
|
||||
pending.reject(new Error(`${pending.method}: ${message.error.message ?? "Unknown CDP error"}`));
|
||||
return;
|
||||
}
|
||||
pending.resolve(message.result);
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof message.sessionId === "string" && typeof message.method === "string") {
|
||||
const session = this.sessions.get(message.sessionId);
|
||||
if (session) {
|
||||
session.handleEvent(message.method, (message.params ?? {}) as JsonObject);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async sendCommand<T>(
|
||||
method: string,
|
||||
params: JsonObject = {},
|
||||
sessionId?: string,
|
||||
): Promise<T> {
|
||||
const id = this.nextId;
|
||||
this.nextId += 1;
|
||||
|
||||
const payload = sessionId ? { id, method, params, sessionId } : { id, method, params };
|
||||
|
||||
const result = new Promise<T>((resolve, reject) => {
|
||||
this.pending.set(id, {
|
||||
resolve: (value) => resolve(value as T),
|
||||
reject,
|
||||
method,
|
||||
});
|
||||
});
|
||||
|
||||
this.ws.send(JSON.stringify(payload));
|
||||
return result;
|
||||
}
|
||||
|
||||
async sendBrowserCommand<T>(method: string, params: JsonObject = {}): Promise<T> {
|
||||
return this.sendCommand<T>(method, params);
|
||||
}
|
||||
|
||||
async sendSessionCommand<T>(sessionId: string, method: string, params: JsonObject = {}): Promise<T> {
|
||||
return this.sendCommand<T>(method, params, sessionId);
|
||||
}
|
||||
|
||||
private async createPageTarget(initialUrl: string, visible = false): Promise<{ targetId: string }> {
|
||||
const attempts: JsonObject[] = visible
|
||||
? [
|
||||
{
|
||||
url: initialUrl,
|
||||
newWindow: true,
|
||||
focus: true,
|
||||
},
|
||||
{
|
||||
url: initialUrl,
|
||||
focus: true,
|
||||
},
|
||||
{
|
||||
url: initialUrl,
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
url: initialUrl,
|
||||
hidden: true,
|
||||
},
|
||||
{
|
||||
url: initialUrl,
|
||||
background: true,
|
||||
focus: false,
|
||||
},
|
||||
{
|
||||
url: initialUrl,
|
||||
},
|
||||
];
|
||||
|
||||
let lastError: unknown;
|
||||
|
||||
for (const params of attempts) {
|
||||
try {
|
||||
return await this.sendBrowserCommand<{ targetId: string }>("Target.createTarget", params);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error("Target.createTarget failed");
|
||||
}
|
||||
|
||||
async createPageSession(options: CreatePageSessionOptions = {}): Promise<TargetSession> {
|
||||
const initialUrl = options.initialUrl ?? "about:blank";
|
||||
const created = await this.createPageTarget(initialUrl, Boolean(options.visible));
|
||||
const attached = await this.sendBrowserCommand<{ sessionId: string }>("Target.attachToTarget", {
|
||||
targetId: created.targetId,
|
||||
flatten: true,
|
||||
});
|
||||
|
||||
const session = new TargetSession(this, created.targetId, attached.sessionId);
|
||||
this.sessions.set(attached.sessionId, session);
|
||||
|
||||
if (options.visible) {
|
||||
await this.sendBrowserCommand("Target.activateTarget", {
|
||||
targetId: created.targetId,
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
await session.send("Page.enable");
|
||||
await session.send("Runtime.enable");
|
||||
await session.send("DOM.enable");
|
||||
|
||||
if (options.visible) {
|
||||
await session.send("Page.bringToFront").catch(() => {});
|
||||
}
|
||||
|
||||
return session;
|
||||
}
|
||||
|
||||
async closeTarget(targetId: string): Promise<void> {
|
||||
try {
|
||||
await this.sendBrowserCommand("Target.closeTarget", { targetId });
|
||||
} catch {
|
||||
// Target may already be gone.
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await new Promise<void>((resolve) => {
|
||||
if (this.ws.readyState === WebSocket.CLOSED) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
this.ws.once("close", () => resolve());
|
||||
this.ws.close();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function evaluateRuntime<T>(session: TargetSession, expression: string): Promise<T> {
|
||||
const response = await session.send<CdpCommandResult<{ value?: T; description?: string }>>("Runtime.evaluate", {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
|
||||
if (response.error) {
|
||||
throw new Error(response.error.message ?? "Runtime.evaluate failed");
|
||||
}
|
||||
|
||||
return (response.result?.value as T | undefined) ?? (undefined as T);
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
import { launch, type LaunchedChrome } from "chrome-launcher";
|
||||
import WebSocket from "ws";
|
||||
import type { Logger } from "../utils/logger";
|
||||
import {
|
||||
cleanChromeLockArtifacts,
|
||||
ensureChromeProfileDir,
|
||||
findChromeProcessUsingProfile,
|
||||
findExistingChromeDebugPort,
|
||||
hasChromeLockArtifacts,
|
||||
listChromeProfileEntries,
|
||||
resolveChromeProfileDir,
|
||||
shouldRetryChromeLaunchRecovery,
|
||||
} from "./profile";
|
||||
|
||||
interface ChromeVersionResponse {
|
||||
webSocketDebuggerUrl: string;
|
||||
}
|
||||
|
||||
export interface ChromeConnectOptions {
|
||||
cdpUrl?: string;
|
||||
browserPath?: string;
|
||||
headless?: boolean;
|
||||
logger?: Logger;
|
||||
profileDir?: string;
|
||||
}
|
||||
|
||||
export interface ChromeConnection {
|
||||
browserWsUrl: string;
|
||||
origin?: string;
|
||||
port?: number;
|
||||
profileDir?: string;
|
||||
launched: boolean;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
async function connectToHttpEndpoint(origin: string): Promise<ChromeConnection> {
|
||||
const normalizedOrigin = origin.replace(/\/$/, "");
|
||||
const version = await fetchJson<ChromeVersionResponse>(`${normalizedOrigin}/json/version`);
|
||||
return {
|
||||
browserWsUrl: version.webSocketDebuggerUrl,
|
||||
origin: normalizedOrigin,
|
||||
port: Number(new URL(normalizedOrigin).port || 80),
|
||||
launched: false,
|
||||
async close() {
|
||||
// Reused external Chrome, nothing to close here.
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function tryReuseChrome(profileDir: string, logger?: Logger): Promise<ChromeConnection | null> {
|
||||
const port = await findExistingChromeDebugPort({ profileDir });
|
||||
if (!port) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const origin = `http://127.0.0.1:${port}`;
|
||||
try {
|
||||
const connection = await connectToHttpEndpoint(origin);
|
||||
logger?.info(`Reusing Chrome debugger at ${origin} for profile ${profileDir}`);
|
||||
return {
|
||||
...connection,
|
||||
profileDir,
|
||||
};
|
||||
} catch {
|
||||
// Debugger disappeared between detection and connect.
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function launchFreshChrome(
|
||||
profileDir: string,
|
||||
options: Pick<ChromeConnectOptions, "browserPath" | "headless">,
|
||||
): Promise<ChromeConnection> {
|
||||
let launchedChrome: LaunchedChrome | null = null;
|
||||
try {
|
||||
launchedChrome = await launch({
|
||||
chromePath: options.browserPath,
|
||||
userDataDir: profileDir,
|
||||
chromeFlags: [
|
||||
"--disable-background-networking",
|
||||
"--disable-default-apps",
|
||||
"--disable-popup-blocking",
|
||||
"--disable-sync",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--remote-allow-origins=*",
|
||||
...(!options.headless ? ["--no-startup-window"] : []),
|
||||
...(options.headless ? ["--headless=new"] : []),
|
||||
],
|
||||
});
|
||||
|
||||
const origin = `http://127.0.0.1:${launchedChrome.port}`;
|
||||
const version = await fetchJson<ChromeVersionResponse>(`${origin}/json/version`);
|
||||
|
||||
const chrome = launchedChrome;
|
||||
return {
|
||||
browserWsUrl: version.webSocketDebuggerUrl,
|
||||
origin,
|
||||
port: launchedChrome.port,
|
||||
profileDir,
|
||||
launched: true,
|
||||
async close() {
|
||||
if (!chrome) return;
|
||||
await gracefulCloseChrome(chrome, origin);
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
launchedChrome?.kill();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function gracefulCloseChrome(chrome: LaunchedChrome, origin: string): Promise<void> {
|
||||
try {
|
||||
const resp = await fetch(`${origin}/json/version`);
|
||||
const { webSocketDebuggerUrl } = (await resp.json()) as ChromeVersionResponse;
|
||||
if (webSocketDebuggerUrl) {
|
||||
const ws = await new Promise<WebSocket>((resolve, reject) => {
|
||||
const socket = new WebSocket(webSocketDebuggerUrl);
|
||||
socket.once("open", () => resolve(socket));
|
||||
socket.once("error", reject);
|
||||
});
|
||||
const id = 1;
|
||||
ws.send(JSON.stringify({ id, method: "Browser.close" }));
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(() => { ws.close(); resolve(); }, 5_000);
|
||||
ws.once("close", () => { clearTimeout(timer); resolve(); });
|
||||
});
|
||||
const exited = await new Promise<boolean>((resolve) => {
|
||||
if (chrome.pid && !isProcessAlive(chrome.pid)) { resolve(true); return; }
|
||||
const timer = setTimeout(() => resolve(false), 3_000);
|
||||
chrome.process.once("exit", () => { clearTimeout(timer); resolve(true); });
|
||||
});
|
||||
if (exited) return;
|
||||
}
|
||||
} catch {}
|
||||
chrome.kill();
|
||||
}
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
try { process.kill(pid, 0); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
export async function connectChrome(options: ChromeConnectOptions): Promise<ChromeConnection> {
|
||||
if (options.cdpUrl) {
|
||||
if (options.cdpUrl.startsWith("ws://") || options.cdpUrl.startsWith("wss://")) {
|
||||
return {
|
||||
browserWsUrl: options.cdpUrl,
|
||||
launched: false,
|
||||
async close() {},
|
||||
};
|
||||
}
|
||||
return connectToHttpEndpoint(options.cdpUrl);
|
||||
}
|
||||
|
||||
const profileDir = ensureChromeProfileDir(resolveChromeProfileDir(options.profileDir));
|
||||
const reused = await tryReuseChrome(profileDir, options.logger);
|
||||
if (reused) {
|
||||
return reused;
|
||||
}
|
||||
|
||||
options.logger?.warn(`No running Chrome debugger found for profile ${profileDir}. Launching Chrome with that profile.`);
|
||||
try {
|
||||
return await launchFreshChrome(profileDir, options);
|
||||
} catch (error) {
|
||||
const entries = await listChromeProfileEntries(profileDir);
|
||||
const shouldRetry = shouldRetryChromeLaunchRecovery({
|
||||
hasLockArtifacts: hasChromeLockArtifacts(entries),
|
||||
hasLiveOwner: findChromeProcessUsingProfile(profileDir),
|
||||
});
|
||||
if (!shouldRetry) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
options.logger?.warn(`Chrome launch failed with stale profile locks. Cleaning ${profileDir} and retrying once.`);
|
||||
cleanChromeLockArtifacts(profileDir);
|
||||
return await launchFreshChrome(profileDir, options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { resolveChromeProfileDir } from "./profile";
|
||||
import type { TargetSession } from "./cdp-client";
|
||||
|
||||
export interface CdpCookie {
|
||||
name: string;
|
||||
value: string;
|
||||
domain: string;
|
||||
path: string;
|
||||
expires: number;
|
||||
size: number;
|
||||
httpOnly: boolean;
|
||||
secure: boolean;
|
||||
session: boolean;
|
||||
sameSite?: string;
|
||||
priority?: string;
|
||||
sameParty?: boolean;
|
||||
sourceScheme?: string;
|
||||
sourcePort?: number;
|
||||
partitionKey?: string;
|
||||
}
|
||||
|
||||
interface SidecarData {
|
||||
savedAt: string;
|
||||
cookies: CdpCookie[];
|
||||
}
|
||||
|
||||
export interface CookieSidecarConfig {
|
||||
urls: readonly string[];
|
||||
filename: string;
|
||||
requiredCookieNames: readonly string[];
|
||||
filterCookie?: (cookie: CdpCookie) => boolean;
|
||||
}
|
||||
|
||||
function sidecarPath(filename: string, profileDir?: string): string {
|
||||
return join(resolveChromeProfileDir(profileDir), filename);
|
||||
}
|
||||
|
||||
function hasRequired(cookies: CdpCookie[], names: readonly string[]): boolean {
|
||||
return names.every((name) =>
|
||||
cookies.some((c) => c.name === name && Boolean(c.value)),
|
||||
);
|
||||
}
|
||||
|
||||
async function getCookies(session: TargetSession, urls: readonly string[]): Promise<CdpCookie[]> {
|
||||
const { cookies } = await session.send<{ cookies: CdpCookie[] }>(
|
||||
"Network.getCookies",
|
||||
{ urls: [...urls] },
|
||||
);
|
||||
return cookies ?? [];
|
||||
}
|
||||
|
||||
export async function exportCookies(
|
||||
session: TargetSession,
|
||||
config: CookieSidecarConfig,
|
||||
profileDir?: string,
|
||||
): Promise<boolean> {
|
||||
const all = await getCookies(session, config.urls);
|
||||
const filtered = config.filterCookie ? all.filter(config.filterCookie) : all;
|
||||
if (!hasRequired(filtered, config.requiredCookieNames)) return false;
|
||||
|
||||
const filePath = sidecarPath(config.filename, profileDir);
|
||||
await mkdir(dirname(filePath), { recursive: true });
|
||||
const data: SidecarData = { savedAt: new Date().toISOString(), cookies: filtered };
|
||||
await writeFile(filePath, JSON.stringify(data, null, 2));
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function restoreCookies(
|
||||
session: TargetSession,
|
||||
config: CookieSidecarConfig,
|
||||
profileDir?: string,
|
||||
): Promise<boolean> {
|
||||
const live = await getCookies(session, config.urls);
|
||||
if (hasRequired(live, config.requiredCookieNames)) return false;
|
||||
|
||||
const filePath = sidecarPath(config.filename, profileDir);
|
||||
const raw = await readFile(filePath, "utf8");
|
||||
const data = JSON.parse(raw) as SidecarData;
|
||||
if (!data.cookies?.length) return false;
|
||||
|
||||
const now = Date.now() / 1000;
|
||||
const valid = data.cookies.filter((c) => c.session || !c.expires || c.expires > now);
|
||||
if (!hasRequired(valid, config.requiredCookieNames)) return false;
|
||||
|
||||
await session.send("Network.setCookies", {
|
||||
cookies: valid.map((c) => ({
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
domain: c.domain,
|
||||
path: c.path,
|
||||
httpOnly: c.httpOnly,
|
||||
secure: c.secure,
|
||||
sameSite: c.sameSite,
|
||||
expires: c.expires,
|
||||
})),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import type { WaitForInteractionRequest } from "../adapters/types";
|
||||
import type { BrowserSession } from "./session";
|
||||
|
||||
interface GateSnapshot {
|
||||
title: string;
|
||||
currentUrl: string;
|
||||
bodyText: string;
|
||||
hasCloudflareTurnstile: boolean;
|
||||
hasCloudflareChallenge: boolean;
|
||||
hasRecaptcha: boolean;
|
||||
hasRecaptchaIframe: boolean;
|
||||
hasHcaptcha: boolean;
|
||||
hasHcaptchaIframe: boolean;
|
||||
}
|
||||
|
||||
export function detectInteractionGateFromSnapshot(snapshot: GateSnapshot): WaitForInteractionRequest | null {
|
||||
const text = snapshot.bodyText.toLowerCase();
|
||||
const title = snapshot.title.toLowerCase();
|
||||
const url = snapshot.currentUrl.toLowerCase();
|
||||
|
||||
if (
|
||||
snapshot.hasCloudflareTurnstile ||
|
||||
snapshot.hasCloudflareChallenge ||
|
||||
title.includes("just a moment") ||
|
||||
text.includes("verify you are human") ||
|
||||
text.includes("checking your browser before accessing") ||
|
||||
text.includes("enable javascript and cookies to continue") ||
|
||||
url.includes("/cdn-cgi/challenge-platform/")
|
||||
) {
|
||||
return {
|
||||
type: "wait_for_interaction",
|
||||
kind: "cloudflare",
|
||||
provider: "cloudflare",
|
||||
reason: "Cloudflare human verification detected",
|
||||
prompt: "Please complete the Cloudflare verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
|
||||
requiresVisibleBrowser: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
snapshot.hasRecaptcha ||
|
||||
snapshot.hasRecaptchaIframe ||
|
||||
text.includes("i'm not a robot") ||
|
||||
text.includes("recaptcha")
|
||||
) {
|
||||
return {
|
||||
type: "wait_for_interaction",
|
||||
kind: "recaptcha",
|
||||
provider: "google_recaptcha",
|
||||
reason: "Google reCAPTCHA detected",
|
||||
prompt: "Please complete the reCAPTCHA verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
|
||||
requiresVisibleBrowser: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
snapshot.hasHcaptcha ||
|
||||
snapshot.hasHcaptchaIframe ||
|
||||
text.includes("hcaptcha")
|
||||
) {
|
||||
return {
|
||||
type: "wait_for_interaction",
|
||||
kind: "hcaptcha",
|
||||
provider: "hcaptcha",
|
||||
reason: "hCaptcha verification detected",
|
||||
prompt: "Please complete the hCaptcha verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
|
||||
requiresVisibleBrowser: true,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function detectInteractionGate(browser: BrowserSession): Promise<WaitForInteractionRequest | null> {
|
||||
const snapshot = await browser.evaluate<GateSnapshot>(`
|
||||
(() => {
|
||||
const bodyText = (document.body?.innerText ?? "").slice(0, 4000);
|
||||
return {
|
||||
title: document.title ?? "",
|
||||
currentUrl: window.location.href,
|
||||
bodyText,
|
||||
hasCloudflareTurnstile: Boolean(
|
||||
document.querySelector(
|
||||
'.cf-turnstile, [name="cf-turnstile-response"], iframe[src*="challenges.cloudflare.com"]'
|
||||
)
|
||||
),
|
||||
hasCloudflareChallenge: Boolean(
|
||||
document.querySelector(
|
||||
'#challenge-running, #cf-challenge-running, .challenge-platform, [data-ray], [data-translate="checking_browser"]'
|
||||
)
|
||||
),
|
||||
hasRecaptcha: Boolean(
|
||||
document.querySelector(
|
||||
'.g-recaptcha, textarea[name="g-recaptcha-response"], iframe[title*="reCAPTCHA"]'
|
||||
)
|
||||
),
|
||||
hasRecaptchaIframe: Boolean(
|
||||
document.querySelector('iframe[src*="google.com/recaptcha"], iframe[src*="recaptcha/api2"]')
|
||||
),
|
||||
hasHcaptcha: Boolean(
|
||||
document.querySelector(
|
||||
'.h-captcha, textarea[name="h-captcha-response"], iframe[title*="hCaptcha"]'
|
||||
)
|
||||
),
|
||||
hasHcaptchaIframe: Boolean(
|
||||
document.querySelector('iframe[src*="hcaptcha.com"]')
|
||||
),
|
||||
};
|
||||
})()
|
||||
`).catch(() => ({
|
||||
title: "",
|
||||
currentUrl: "",
|
||||
bodyText: "",
|
||||
hasCloudflareTurnstile: false,
|
||||
hasCloudflareChallenge: false,
|
||||
hasRecaptcha: false,
|
||||
hasRecaptchaIframe: false,
|
||||
hasHcaptcha: false,
|
||||
hasHcaptchaIframe: false,
|
||||
}));
|
||||
|
||||
return detectInteractionGateFromSnapshot(snapshot);
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
import type { TargetSession } from "./cdp-client";
|
||||
import type { Logger } from "../utils/logger";
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
export interface NetworkEntry {
|
||||
requestId: string;
|
||||
url: string;
|
||||
method: string;
|
||||
resourceType: string;
|
||||
timestamp: number;
|
||||
requestHeaders?: Record<string, string>;
|
||||
requestBody?: string;
|
||||
status?: number;
|
||||
statusText?: string;
|
||||
responseHeaders?: Record<string, string>;
|
||||
mimeType?: string;
|
||||
body?: string;
|
||||
bodyBase64?: boolean;
|
||||
bodyError?: string;
|
||||
failed?: boolean;
|
||||
failureReason?: string;
|
||||
finished: boolean;
|
||||
}
|
||||
|
||||
function normalizeHeaders(headers: unknown): Record<string, string> | undefined {
|
||||
if (!headers || typeof headers !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers as Record<string, unknown>).map(([key, value]) => [key, String(value)]),
|
||||
);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export class NetworkJournal {
|
||||
private readonly entries = new Map<string, NetworkEntry>();
|
||||
private lastActivityAt = Date.now();
|
||||
private started = false;
|
||||
|
||||
constructor(
|
||||
private readonly session: TargetSession,
|
||||
private readonly log: Logger,
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.started = true;
|
||||
this.session.on("Network.requestWillBeSent", this.handleRequestWillBeSent);
|
||||
this.session.on("Network.responseReceived", this.handleResponseReceived);
|
||||
this.session.on("Network.loadingFinished", this.handleLoadingFinished);
|
||||
this.session.on("Network.loadingFailed", this.handleLoadingFailed);
|
||||
await this.session.send("Network.enable");
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.started) {
|
||||
return;
|
||||
}
|
||||
this.session.off("Network.requestWillBeSent", this.handleRequestWillBeSent);
|
||||
this.session.off("Network.responseReceived", this.handleResponseReceived);
|
||||
this.session.off("Network.loadingFinished", this.handleLoadingFinished);
|
||||
this.session.off("Network.loadingFailed", this.handleLoadingFailed);
|
||||
this.started = false;
|
||||
}
|
||||
|
||||
private touch(): void {
|
||||
this.lastActivityAt = Date.now();
|
||||
}
|
||||
|
||||
private readonly handleRequestWillBeSent = (params: JsonObject): void => {
|
||||
const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
|
||||
const request = params.request as JsonObject | undefined;
|
||||
if (!requestId || !request) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.touch();
|
||||
this.entries.set(requestId, {
|
||||
requestId,
|
||||
url: String(request.url ?? ""),
|
||||
method: String(request.method ?? "GET"),
|
||||
resourceType: String(params.type ?? "Other"),
|
||||
timestamp: Date.now(),
|
||||
requestHeaders: normalizeHeaders(request.headers),
|
||||
requestBody: typeof request.postData === "string" ? request.postData : undefined,
|
||||
finished: false,
|
||||
});
|
||||
};
|
||||
|
||||
private readonly handleResponseReceived = (params: JsonObject): void => {
|
||||
const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
|
||||
const response = params.response as JsonObject | undefined;
|
||||
if (!requestId || !response) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.touch();
|
||||
const existing = this.entries.get(requestId);
|
||||
if (!existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
existing.status = typeof response.status === "number" ? response.status : undefined;
|
||||
existing.statusText = typeof response.statusText === "string" ? response.statusText : undefined;
|
||||
existing.responseHeaders = normalizeHeaders(response.headers);
|
||||
existing.mimeType = typeof response.mimeType === "string" ? response.mimeType : undefined;
|
||||
this.entries.set(requestId, existing);
|
||||
};
|
||||
|
||||
private readonly handleLoadingFinished = (params: JsonObject): void => {
|
||||
const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
|
||||
if (!requestId) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.touch();
|
||||
const existing = this.entries.get(requestId);
|
||||
if (!existing) {
|
||||
return;
|
||||
}
|
||||
existing.finished = true;
|
||||
this.entries.set(requestId, existing);
|
||||
};
|
||||
|
||||
private readonly handleLoadingFailed = (params: JsonObject): void => {
|
||||
const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
|
||||
if (!requestId) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.touch();
|
||||
const existing = this.entries.get(requestId);
|
||||
if (!existing) {
|
||||
return;
|
||||
}
|
||||
existing.finished = true;
|
||||
existing.failed = true;
|
||||
existing.failureReason = typeof params.errorText === "string" ? params.errorText : "Unknown error";
|
||||
this.entries.set(requestId, existing);
|
||||
};
|
||||
|
||||
getEntries(): NetworkEntry[] {
|
||||
return Array.from(this.entries.values());
|
||||
}
|
||||
|
||||
findEntries(predicate: (entry: NetworkEntry) => boolean): NetworkEntry[] {
|
||||
return this.getEntries().filter(predicate);
|
||||
}
|
||||
|
||||
async waitForIdle(options: { idleMs?: number; timeoutMs?: number } = {}): Promise<void> {
|
||||
const idleMs = options.idleMs ?? 1_200;
|
||||
const timeoutMs = options.timeoutMs ?? 15_000;
|
||||
const startedAt = Date.now();
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (Date.now() - this.lastActivityAt >= idleMs) {
|
||||
return;
|
||||
}
|
||||
await sleep(Math.min(150, idleMs));
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for network idle");
|
||||
}
|
||||
|
||||
async waitForResponse(
|
||||
predicate: (entry: NetworkEntry) => boolean,
|
||||
options: { timeoutMs?: number } = {},
|
||||
): Promise<NetworkEntry> {
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const startedAt = Date.now();
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
const matched = this.getEntries().find((entry) => entry.finished && predicate(entry));
|
||||
if (matched) {
|
||||
return matched;
|
||||
}
|
||||
await sleep(150);
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for matching network response");
|
||||
}
|
||||
|
||||
async ensureBody(entry: NetworkEntry): Promise<string | undefined> {
|
||||
if (entry.body !== undefined) {
|
||||
return entry.body;
|
||||
}
|
||||
if (entry.bodyError || entry.failed || !entry.finished) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.session.send<{ body: string; base64Encoded: boolean }>("Network.getResponseBody", {
|
||||
requestId: entry.requestId,
|
||||
});
|
||||
entry.bodyBase64 = result.base64Encoded;
|
||||
entry.body = result.base64Encoded ? Buffer.from(result.body, "base64").toString("utf8") : result.body;
|
||||
return entry.body;
|
||||
} catch (error) {
|
||||
entry.bodyError = error instanceof Error ? error.message : String(error);
|
||||
this.log.debug(`Failed to fetch response body for ${entry.url}: ${entry.bodyError}`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async getJsonBody(entry: NetworkEntry): Promise<unknown | null> {
|
||||
const body = await this.ensureBody(entry);
|
||||
if (!body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(body);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async toJSON(options: { includeBodies?: boolean } = {}): Promise<NetworkEntry[]> {
|
||||
const entries = this.getEntries();
|
||||
if (!options.includeBodies) {
|
||||
return entries;
|
||||
}
|
||||
|
||||
await Promise.all(entries.map((entry) => this.ensureBody(entry)));
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import type { BrowserSession } from "./session";
|
||||
|
||||
export interface CapturedPageSnapshot {
|
||||
html: string;
|
||||
finalUrl: string;
|
||||
}
|
||||
|
||||
export const CAPTURE_NORMALIZED_PAGE_SCRIPT = String.raw`
|
||||
(() => {
|
||||
const baseUrl = document.baseURI || location.href;
|
||||
const htmlClone = document.documentElement.cloneNode(true);
|
||||
|
||||
function materializeShadowDom(sourceRoot, cloneRoot) {
|
||||
const sourceElements = Array.from(sourceRoot.querySelectorAll("*"));
|
||||
const cloneElements = Array.from(cloneRoot.querySelectorAll("*"));
|
||||
|
||||
for (let index = sourceElements.length - 1; index >= 0; index -= 1) {
|
||||
const sourceElement = sourceElements[index];
|
||||
const cloneElement = cloneElements[index];
|
||||
const shadowRoot = sourceElement && sourceElement.shadowRoot;
|
||||
if (!shadowRoot || !cloneElement || !shadowRoot.innerHTML) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cloneElement.tagName && cloneElement.tagName.includes("-")) {
|
||||
const wrapper = document.createElement("div");
|
||||
wrapper.setAttribute("data-shadow-host", cloneElement.tagName.toLowerCase());
|
||||
wrapper.innerHTML = shadowRoot.innerHTML;
|
||||
cloneElement.replaceWith(wrapper);
|
||||
} else {
|
||||
cloneElement.innerHTML = shadowRoot.innerHTML;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function toAbsolute(url) {
|
||||
if (!url) return url;
|
||||
try {
|
||||
return new URL(url, baseUrl).href;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
function absolutizeAttribute(root, selector, attribute) {
|
||||
root.querySelectorAll(selector).forEach((element) => {
|
||||
const value = element.getAttribute(attribute);
|
||||
if (!value) return;
|
||||
const absolute = toAbsolute(value);
|
||||
if (absolute) {
|
||||
element.setAttribute(attribute, absolute);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function absolutizeSrcset(root, selector) {
|
||||
root.querySelectorAll(selector).forEach((element) => {
|
||||
const srcset = element.getAttribute("srcset");
|
||||
if (!srcset) return;
|
||||
element.setAttribute(
|
||||
"srcset",
|
||||
srcset
|
||||
.split(",")
|
||||
.map((part) => {
|
||||
const trimmed = part.trim();
|
||||
if (!trimmed) return "";
|
||||
const [url, ...descriptor] = trimmed.split(/\s+/);
|
||||
const absolute = toAbsolute(url);
|
||||
return descriptor.length > 0 ? absolute + " " + descriptor.join(" ") : absolute;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join(", "),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
materializeShadowDom(document.documentElement, htmlClone);
|
||||
|
||||
htmlClone
|
||||
.querySelectorAll("img[data-src], video[data-src], audio[data-src], source[data-src]")
|
||||
.forEach((element) => {
|
||||
const dataSource = element.getAttribute("data-src");
|
||||
const current = element.getAttribute("src");
|
||||
if (dataSource && (!current || current === "" || current.startsWith("data:"))) {
|
||||
element.setAttribute("src", dataSource);
|
||||
}
|
||||
});
|
||||
|
||||
absolutizeAttribute(htmlClone, "a[href]", "href");
|
||||
absolutizeAttribute(htmlClone, "img[src], video[src], audio[src], source[src], iframe[src]", "src");
|
||||
absolutizeAttribute(htmlClone, "video[poster]", "poster");
|
||||
absolutizeSrcset(htmlClone, "img[srcset], source[srcset]");
|
||||
|
||||
return {
|
||||
html: "<!doctype html>\n" + htmlClone.outerHTML,
|
||||
finalUrl: location.href,
|
||||
};
|
||||
})()
|
||||
`;
|
||||
|
||||
export async function captureNormalizedPageSnapshot(
|
||||
browser: BrowserSession,
|
||||
): Promise<CapturedPageSnapshot> {
|
||||
return browser.evaluate<CapturedPageSnapshot>(CAPTURE_NORMALIZED_PAGE_SCRIPT);
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
export interface ResolveSharedChromeProfileDirOptions {
|
||||
envNames?: string[];
|
||||
appDataDirName?: string;
|
||||
profileDirName?: string;
|
||||
}
|
||||
|
||||
export interface FindExistingChromeDebugPortOptions {
|
||||
profileDir: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
interface ChromeVersionResponse {
|
||||
webSocketDebuggerUrl?: string;
|
||||
}
|
||||
|
||||
const CHROME_LOCK_FILE_NAMES = ["SingletonLock", "SingletonSocket", "SingletonCookie", "chrome.pid"] as const;
|
||||
|
||||
function resolveDataBaseDir(): string {
|
||||
if (process.platform === "darwin") {
|
||||
return path.join(os.homedir(), "Library", "Application Support");
|
||||
}
|
||||
if (process.platform === "win32") {
|
||||
return process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming");
|
||||
}
|
||||
return process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share");
|
||||
}
|
||||
|
||||
export function resolveSharedChromeProfileDir(
|
||||
options: ResolveSharedChromeProfileDirOptions = {},
|
||||
): string {
|
||||
for (const envName of options.envNames ?? []) {
|
||||
const override = process.env[envName]?.trim();
|
||||
if (override) {
|
||||
return path.resolve(override);
|
||||
}
|
||||
}
|
||||
|
||||
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
|
||||
const profileDirName = options.profileDirName ?? "chrome-profile";
|
||||
return path.join(resolveDataBaseDir(), appDataDirName, profileDirName);
|
||||
}
|
||||
|
||||
export function resolveChromeProfileDir(profileDir?: string): string {
|
||||
if (profileDir?.trim()) {
|
||||
return path.resolve(profileDir.trim());
|
||||
}
|
||||
|
||||
return resolveSharedChromeProfileDir({
|
||||
envNames: ["BAOYU_CHROME_PROFILE_DIR"],
|
||||
appDataDirName: "baoyu-skills",
|
||||
profileDirName: "chrome-profile",
|
||||
});
|
||||
}
|
||||
|
||||
export function ensureChromeProfileDir(profileDir: string): string {
|
||||
fs.mkdirSync(profileDir, { recursive: true });
|
||||
return profileDir;
|
||||
}
|
||||
|
||||
export function hasChromeLockArtifacts(entries: readonly string[]): boolean {
|
||||
return CHROME_LOCK_FILE_NAMES.some((name) => entries.includes(name));
|
||||
}
|
||||
|
||||
export function shouldRetryChromeLaunchRecovery(options: {
|
||||
hasLockArtifacts: boolean;
|
||||
hasLiveOwner: boolean;
|
||||
}): boolean {
|
||||
return options.hasLockArtifacts && !options.hasLiveOwner;
|
||||
}
|
||||
|
||||
export function findChromeProcessUsingProfile(profileDir: string): boolean {
|
||||
if (process.platform === "win32") {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = spawnSync("ps", ["aux"], {
|
||||
encoding: "utf8",
|
||||
timeout: 5_000,
|
||||
});
|
||||
if (result.status !== 0 || !result.stdout) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return result.stdout
|
||||
.split("\n")
|
||||
.some((line) => line.includes(`--user-data-dir=${profileDir}`));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function cleanChromeLockArtifacts(profileDir: string): void {
|
||||
for (const name of CHROME_LOCK_FILE_NAMES) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(profileDir, name));
|
||||
} catch {
|
||||
// Ignore missing files and continue cleaning the remaining artifacts.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function listChromeProfileEntries(profileDir: string): Promise<string[]> {
|
||||
try {
|
||||
return await fs.promises.readdir(profileDir);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(url: string, timeoutMs = 3_000): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fetch(url, {
|
||||
redirect: "follow",
|
||||
signal: controller.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string, timeoutMs = 3_000): Promise<T> {
|
||||
const response = await fetchWithTimeout(url, timeoutMs);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
|
||||
try {
|
||||
const version = await fetchJson<ChromeVersionResponse>(`http://127.0.0.1:${port}/json/version`, timeoutMs);
|
||||
return Boolean(version.webSocketDebuggerUrl);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function parseDevToolsActivePort(filePath: string): { port: number; wsPath: string } | null {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, "utf8");
|
||||
const lines = content.split(/\r?\n/);
|
||||
const port = Number.parseInt(lines[0]?.trim() ?? "", 10);
|
||||
const wsPath = lines[1]?.trim() ?? "";
|
||||
if (port > 0 && wsPath) {
|
||||
return { port, wsPath };
|
||||
}
|
||||
} catch {
|
||||
// Ignore and fall back to process inspection.
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function findExistingChromeDebugPort(
|
||||
options: FindExistingChromeDebugPortOptions,
|
||||
): Promise<number | null> {
|
||||
const timeoutMs = options.timeoutMs ?? 3_000;
|
||||
const activePort = parseDevToolsActivePort(path.join(options.profileDir, "DevToolsActivePort"));
|
||||
if (activePort && await isDebugPortReady(activePort.port, timeoutMs)) {
|
||||
return activePort.port;
|
||||
}
|
||||
|
||||
if (process.platform === "win32") {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = spawnSync("ps", ["aux"], {
|
||||
encoding: "utf8",
|
||||
timeout: 5_000,
|
||||
});
|
||||
if (result.status !== 0 || !result.stdout) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const lines = result.stdout
|
||||
.split("\n")
|
||||
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
|
||||
|
||||
for (const line of lines) {
|
||||
const match = line.match(/--remote-debugging-port=(\d+)/);
|
||||
const port = Number.parseInt(match?.[1] ?? "", 10);
|
||||
if (port > 0 && await isDebugPortReady(port, timeoutMs)) {
|
||||
return port;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore and report no reusable debugger.
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { CdpClient, TargetSession, evaluateRuntime } from "./cdp-client";
|
||||
|
||||
interface NavigationResult {
|
||||
errorText?: string;
|
||||
}
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MACOS_BROWSER_APP_IDS = [
|
||||
"com.google.Chrome",
|
||||
"org.chromium.Chromium",
|
||||
"com.brave.Browser",
|
||||
"com.microsoft.edgemac",
|
||||
];
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function activateBrowserApp(): Promise<void> {
|
||||
if (process.platform !== "darwin") {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const appId of MACOS_BROWSER_APP_IDS) {
|
||||
try {
|
||||
await execFileAsync("osascript", ["-e", `tell application id "${appId}" to activate`]);
|
||||
return;
|
||||
} catch {
|
||||
// Try the next installed browser bundle id.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class BrowserSession {
|
||||
private constructor(
|
||||
private readonly cdp: CdpClient,
|
||||
public readonly targetSession: TargetSession,
|
||||
public readonly interactive: boolean,
|
||||
) {}
|
||||
|
||||
static async open(
|
||||
cdp: CdpClient,
|
||||
options: {
|
||||
initialUrl?: string;
|
||||
interactive?: boolean;
|
||||
} = {},
|
||||
): Promise<BrowserSession> {
|
||||
const targetSession = await cdp.createPageSession({
|
||||
initialUrl: options.initialUrl,
|
||||
visible: options.interactive,
|
||||
});
|
||||
const browser = new BrowserSession(cdp, targetSession, Boolean(options.interactive));
|
||||
if (browser.interactive) {
|
||||
await browser.bringToFront().catch(() => {});
|
||||
}
|
||||
return browser;
|
||||
}
|
||||
|
||||
async goto(url: string, timeoutMs = 30_000): Promise<void> {
|
||||
const loadPromise = this.targetSession.waitForEvent("Page.loadEventFired", undefined, timeoutMs).catch(() => null);
|
||||
const result = await this.targetSession.send<NavigationResult>("Page.navigate", { url });
|
||||
if (result.errorText) {
|
||||
throw new Error(`Navigation failed: ${result.errorText}`);
|
||||
}
|
||||
await loadPromise;
|
||||
await this.waitForReadyState(timeoutMs);
|
||||
}
|
||||
|
||||
async waitForReadyState(timeoutMs = 30_000): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
const state = await this.evaluate<string>("document.readyState");
|
||||
if (state === "interactive" || state === "complete") {
|
||||
return;
|
||||
}
|
||||
await sleep(150);
|
||||
}
|
||||
throw new Error("Timed out waiting for document.readyState");
|
||||
}
|
||||
|
||||
async evaluate<T>(expression: string): Promise<T> {
|
||||
return evaluateRuntime<T>(this.targetSession, expression);
|
||||
}
|
||||
|
||||
async getHTML(): Promise<string> {
|
||||
return this.evaluate<string>("document.documentElement.outerHTML");
|
||||
}
|
||||
|
||||
async getTitle(): Promise<string> {
|
||||
return this.evaluate<string>("document.title");
|
||||
}
|
||||
|
||||
async getURL(): Promise<string> {
|
||||
return this.evaluate<string>("window.location.href");
|
||||
}
|
||||
|
||||
async bringToFront(): Promise<void> {
|
||||
await this.targetSession.send("Page.bringToFront").catch(async () => {
|
||||
await this.cdp.sendBrowserCommand("Target.activateTarget", {
|
||||
targetId: this.targetSession.targetId,
|
||||
});
|
||||
});
|
||||
if (this.interactive) {
|
||||
await activateBrowserApp().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async click(selector: string): Promise<void> {
|
||||
const result = await this.evaluate<{ ok: boolean; error?: string }>(`
|
||||
(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) {
|
||||
return { ok: false, error: "Element not found" };
|
||||
}
|
||||
element.scrollIntoView({ block: "center", inline: "center" });
|
||||
if (element instanceof HTMLElement) {
|
||||
element.click();
|
||||
return { ok: true };
|
||||
}
|
||||
return { ok: false, error: "Element is not clickable" };
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(result.error ?? `Failed to click ${selector}`);
|
||||
}
|
||||
}
|
||||
|
||||
async scrollToEnd(options: { stepPx?: number; delayMs?: number; maxSteps?: number } = {}): Promise<void> {
|
||||
const stepPx = options.stepPx ?? 1_400;
|
||||
const delayMs = options.delayMs ?? 250;
|
||||
const maxSteps = options.maxSteps ?? 6;
|
||||
|
||||
for (let step = 0; step < maxSteps; step += 1) {
|
||||
const done = await this.evaluate<boolean>(`
|
||||
(() => {
|
||||
const before = window.scrollY;
|
||||
window.scrollBy(0, ${stepPx});
|
||||
const atBottom = window.innerHeight + window.scrollY >= document.body.scrollHeight - 4;
|
||||
return atBottom || window.scrollY === before;
|
||||
})()
|
||||
`);
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.cdp.closeTarget(this.targetSession.targetId);
|
||||
}
|
||||
}
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import {
|
||||
runConvertCommand,
|
||||
type ConvertCommandOptions,
|
||||
type OutputFormat,
|
||||
type WaitMode,
|
||||
} from "./commands/convert";
|
||||
|
||||
export const HELP_TEXT = `
|
||||
baoyu-fetch - Read a URL into Markdown or JSON with Chrome CDP
|
||||
|
||||
Usage:
|
||||
baoyu-fetch <url> [options]
|
||||
|
||||
Options:
|
||||
--output <file> Save output to file
|
||||
--format <type> Output format: markdown | json
|
||||
--json Alias for --format json
|
||||
--adapter <name> Force an adapter (e.g. x, generic)
|
||||
--download-media Download adapter-reported media into ./imgs and ./videos, then rewrite markdown links
|
||||
--media-dir <dir> Base directory for downloaded media. Defaults to the output directory
|
||||
--debug-dir <dir> Write debug artifacts
|
||||
--cdp-url <url> Reuse an existing Chrome DevTools endpoint
|
||||
--browser-path <path> Explicit Chrome binary path
|
||||
--chrome-profile-dir <path>
|
||||
Chrome user data dir. Defaults to BAOYU_CHROME_PROFILE_DIR
|
||||
or baoyu-skills/chrome-profile.
|
||||
--headless Launch a temporary headless Chrome if needed
|
||||
--wait-for <mode> Wait mode: interaction | force
|
||||
interaction: start visible Chrome and auto-wait only when login or verification is required
|
||||
force: start visible Chrome, then auto-continue after it detects login/challenge progress
|
||||
or continue immediately when you press Enter
|
||||
--wait-for-interaction
|
||||
Alias for --wait-for interaction
|
||||
--wait-for-login Alias for --wait-for interaction
|
||||
--interaction-timeout <ms>
|
||||
How long to wait for manual interaction before failing (default: 600000)
|
||||
--interaction-poll-interval <ms>
|
||||
How often to poll interaction state while waiting (default: 1500)
|
||||
--login-timeout <ms> Alias for --interaction-timeout
|
||||
--login-poll-interval <ms>
|
||||
Alias for --interaction-poll-interval
|
||||
--timeout <ms> Page timeout in milliseconds (default: 30000)
|
||||
--help Show help
|
||||
|
||||
Examples:
|
||||
baoyu-fetch https://example.com
|
||||
baoyu-fetch https://example.com --format markdown --output article.md --download-media
|
||||
baoyu-fetch https://example.com --format json --output article.json
|
||||
baoyu-fetch https://x.com/lennysan/status/2036483059407810640 --wait-for interaction
|
||||
baoyu-fetch https://x.com/lennysan/status/2036483059407810640 --wait-for force
|
||||
`.trim();
|
||||
|
||||
interface CliOptions extends ConvertCommandOptions {
|
||||
url?: string;
|
||||
help: boolean;
|
||||
}
|
||||
|
||||
function normalizeWaitMode(raw: string): WaitMode {
|
||||
const value = raw.toLowerCase();
|
||||
if (value === "interaction" || value === "auto") {
|
||||
return "interaction";
|
||||
}
|
||||
if (value === "force" || value === "manual" || value === "always") {
|
||||
return "force";
|
||||
}
|
||||
throw new Error(`Invalid wait mode: ${raw}. Expected interaction or force.`);
|
||||
}
|
||||
|
||||
function normalizeOutputFormat(raw: string): OutputFormat {
|
||||
const value = raw.toLowerCase();
|
||||
if (value === "markdown" || value === "json") {
|
||||
return value;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid output format: ${raw}. Expected markdown or json.`);
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): CliOptions {
|
||||
const options: CliOptions = {
|
||||
format: "markdown",
|
||||
headless: false,
|
||||
downloadMedia: false,
|
||||
waitMode: "none",
|
||||
interactionTimeoutMs: 600_000,
|
||||
interactionPollIntervalMs: 1_500,
|
||||
timeoutMs: 30_000,
|
||||
help: false,
|
||||
};
|
||||
|
||||
const args = argv.slice(2);
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const value = args[index];
|
||||
|
||||
if (value === "--help" || value === "-h") {
|
||||
options.help = true;
|
||||
continue;
|
||||
}
|
||||
if (value === "--format") {
|
||||
const format = args[index + 1];
|
||||
if (!format) {
|
||||
throw new Error("--format requires a value");
|
||||
}
|
||||
options.format = normalizeOutputFormat(format);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--json") {
|
||||
options.format = "json";
|
||||
continue;
|
||||
}
|
||||
if (value === "--download-media") {
|
||||
options.downloadMedia = true;
|
||||
continue;
|
||||
}
|
||||
if (value === "--headless") {
|
||||
options.headless = true;
|
||||
continue;
|
||||
}
|
||||
if (value === "--wait-for") {
|
||||
const mode = args[index + 1];
|
||||
if (!mode) {
|
||||
throw new Error("--wait-for requires a mode");
|
||||
}
|
||||
options.waitMode = normalizeWaitMode(mode);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--wait-for-interaction" || value === "--wait-for-login") {
|
||||
options.waitMode = "interaction";
|
||||
continue;
|
||||
}
|
||||
if (value === "--output") {
|
||||
options.output = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--adapter") {
|
||||
options.adapter = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--debug-dir") {
|
||||
options.debugDir = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--media-dir") {
|
||||
options.mediaDir = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--cdp-url") {
|
||||
options.cdpUrl = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--browser-path") {
|
||||
options.browserPath = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--chrome-profile-dir") {
|
||||
options.chromeProfileDir = args[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--timeout") {
|
||||
const parsed = Number(args[index + 1]);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid timeout: ${args[index + 1]}`);
|
||||
}
|
||||
options.timeoutMs = parsed;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--interaction-timeout" || value === "--login-timeout") {
|
||||
const parsed = Number(args[index + 1]);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid interaction timeout: ${args[index + 1]}`);
|
||||
}
|
||||
options.interactionTimeoutMs = parsed;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === "--interaction-poll-interval" || value === "--login-poll-interval") {
|
||||
const parsed = Number(args[index + 1]);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid interaction poll interval: ${args[index + 1]}`);
|
||||
}
|
||||
options.interactionPollIntervalMs = parsed;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (value.startsWith("-")) {
|
||||
throw new Error(`Unknown option: ${value}`);
|
||||
}
|
||||
if (!options.url) {
|
||||
options.url = value;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unexpected argument: ${value}`);
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
try {
|
||||
const options = parseArgs(process.argv);
|
||||
if (options.help || !options.url) {
|
||||
console.log(HELP_TEXT);
|
||||
return;
|
||||
}
|
||||
|
||||
await runConvertCommand(options);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(message);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
void main();
|
||||
}
|
||||
@@ -0,0 +1,580 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { connectChrome, type ChromeConnection } from "../browser/chrome-launcher";
|
||||
import { CdpClient } from "../browser/cdp-client";
|
||||
import { detectInteractionGate } from "../browser/interaction-gates";
|
||||
import { NetworkJournal } from "../browser/network-journal";
|
||||
import { BrowserSession } from "../browser/session";
|
||||
import { genericAdapter, resolveAdapter } from "../adapters";
|
||||
import { isXSessionReady } from "../adapters/x/session";
|
||||
import type { ExtractedDocument } from "../extract/document";
|
||||
import { renderMarkdown } from "../extract/markdown-renderer";
|
||||
import { downloadMediaAssets } from "../media/default-downloader";
|
||||
import { rewriteMarkdownMediaLinks } from "../media/markdown-media";
|
||||
import { createLogger } from "../utils/logger";
|
||||
import { normalizeUrl } from "../utils/url";
|
||||
import type {
|
||||
Adapter,
|
||||
AdapterContext,
|
||||
AdapterLoginInfo,
|
||||
LoginState,
|
||||
MediaAsset,
|
||||
WaitForInteractionRequest,
|
||||
} from "../adapters/types";
|
||||
|
||||
export type WaitMode = "none" | "interaction" | "force";
|
||||
export type OutputFormat = "markdown" | "json";
|
||||
|
||||
export interface ConvertCommandOptions {
|
||||
url?: string;
|
||||
output?: string;
|
||||
format: OutputFormat;
|
||||
adapter?: string;
|
||||
debugDir?: string;
|
||||
cdpUrl?: string;
|
||||
browserPath?: string;
|
||||
chromeProfileDir?: string;
|
||||
headless: boolean;
|
||||
downloadMedia: boolean;
|
||||
mediaDir?: string;
|
||||
waitMode: WaitMode;
|
||||
interactionTimeoutMs: number;
|
||||
interactionPollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
interface RuntimeResources {
|
||||
chrome: ChromeConnection;
|
||||
cdp: CdpClient;
|
||||
browser: BrowserSession;
|
||||
network: NetworkJournal;
|
||||
interactive: boolean;
|
||||
}
|
||||
|
||||
interface ForceWaitSnapshot {
|
||||
url: string;
|
||||
hasGate: boolean;
|
||||
loginState: LoginState | "unavailable";
|
||||
sessionReady: boolean;
|
||||
}
|
||||
|
||||
interface SuccessfulConvertOutput {
|
||||
adapter: string;
|
||||
status: "ok";
|
||||
login?: AdapterLoginInfo;
|
||||
media: MediaAsset[];
|
||||
downloads: Awaited<ReturnType<typeof downloadMediaAssets>> | null;
|
||||
document: ExtractedDocument;
|
||||
markdown: string;
|
||||
}
|
||||
|
||||
interface InteractionRequiredOutput {
|
||||
adapter: string;
|
||||
status: "needs_interaction";
|
||||
login?: AdapterLoginInfo;
|
||||
interaction: WaitForInteractionRequest;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function isForceWaitSessionReady(snapshot: ForceWaitSnapshot): boolean {
|
||||
return snapshot.sessionReady;
|
||||
}
|
||||
|
||||
export function shouldKeepBrowserOpenAfterInteraction(options: {
|
||||
launched: boolean;
|
||||
interaction: Pick<WaitForInteractionRequest, "kind" | "provider">;
|
||||
}): boolean {
|
||||
return options.launched && options.interaction.kind === "login" && options.interaction.provider === "x";
|
||||
}
|
||||
|
||||
export function shouldAutoContinueForceWait(
|
||||
initial: ForceWaitSnapshot,
|
||||
current: ForceWaitSnapshot,
|
||||
): boolean {
|
||||
if (initial.hasGate && !current.hasGate) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (initial.loginState === "logged_out" && current.loginState !== "logged_out" && isForceWaitSessionReady(current)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (initial.loginState !== "logged_in" && current.loginState === "logged_in" && isForceWaitSessionReady(current)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
current.url !== initial.url &&
|
||||
!current.hasGate &&
|
||||
current.loginState !== "logged_out" &&
|
||||
isForceWaitSessionReady(current)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function writeOutput(path: string, content: string): Promise<void> {
|
||||
const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
|
||||
if (directory) {
|
||||
await mkdir(directory, { recursive: true });
|
||||
}
|
||||
await writeFile(path, content, "utf8");
|
||||
}
|
||||
|
||||
async function writeDebugArtifacts(
|
||||
debugDir: string,
|
||||
document: ExtractedDocument,
|
||||
markdown: string,
|
||||
browser: BrowserSession,
|
||||
network: NetworkJournal,
|
||||
): Promise<void> {
|
||||
await mkdir(debugDir, { recursive: true });
|
||||
|
||||
const html = await browser.getHTML().catch(() => "");
|
||||
const networkDump = await network.toJSON({ includeBodies: true });
|
||||
|
||||
await Promise.all([
|
||||
writeFile(join(debugDir, "document.json"), JSON.stringify(document, null, 2), "utf8"),
|
||||
writeFile(join(debugDir, "markdown.md"), markdown, "utf8"),
|
||||
writeFile(join(debugDir, "page.html"), html, "utf8"),
|
||||
writeFile(join(debugDir, "network.json"), JSON.stringify(networkDump, null, 2), "utf8"),
|
||||
]);
|
||||
}
|
||||
|
||||
async function openRuntime(
|
||||
options: ConvertCommandOptions,
|
||||
interactive: boolean,
|
||||
debugEnabled: boolean,
|
||||
): Promise<RuntimeResources> {
|
||||
const logger = createLogger(debugEnabled);
|
||||
if (interactive) {
|
||||
logger.info("Opening Chrome in interactive mode.");
|
||||
}
|
||||
const chrome = await connectChrome({
|
||||
cdpUrl: options.cdpUrl,
|
||||
browserPath: options.browserPath,
|
||||
profileDir: options.chromeProfileDir,
|
||||
headless: interactive ? false : options.headless,
|
||||
logger,
|
||||
});
|
||||
|
||||
const cdp = await CdpClient.connect(chrome.browserWsUrl);
|
||||
const browser = await BrowserSession.open(cdp, { interactive });
|
||||
if (interactive) {
|
||||
await browser.bringToFront().catch(() => {});
|
||||
}
|
||||
const network = new NetworkJournal(browser.targetSession, logger);
|
||||
await network.start();
|
||||
|
||||
return {
|
||||
chrome,
|
||||
cdp,
|
||||
browser,
|
||||
network,
|
||||
interactive,
|
||||
};
|
||||
}
|
||||
|
||||
async function closeRuntime(runtime: RuntimeResources | null | undefined): Promise<void> {
|
||||
if (!runtime) {
|
||||
return;
|
||||
}
|
||||
runtime.network.stop();
|
||||
await runtime.browser.close().catch(() => {});
|
||||
await runtime.cdp.close().catch(() => {});
|
||||
await runtime.chrome.close().catch(() => {});
|
||||
}
|
||||
|
||||
async function isInteractionSessionReady(
|
||||
context: AdapterContext,
|
||||
interaction: WaitForInteractionRequest,
|
||||
): Promise<boolean> {
|
||||
if (interaction.provider !== "x") {
|
||||
return true;
|
||||
}
|
||||
return await isXSessionReady(context).catch(() => false);
|
||||
}
|
||||
|
||||
async function reopenInteractiveRuntime(
|
||||
runtime: RuntimeResources,
|
||||
options: ConvertCommandOptions,
|
||||
debugEnabled: boolean,
|
||||
): Promise<RuntimeResources> {
|
||||
if (runtime.interactive) {
|
||||
return runtime;
|
||||
}
|
||||
|
||||
await closeRuntime(runtime);
|
||||
return openRuntime(options, true, debugEnabled);
|
||||
}
|
||||
|
||||
async function captureForceWaitSnapshot(
|
||||
adapter: Adapter,
|
||||
context: AdapterContext,
|
||||
): Promise<ForceWaitSnapshot> {
|
||||
const [gate, url, login] = await Promise.all([
|
||||
detectInteractionGate(context.browser).catch(() => null),
|
||||
context.browser.getURL().catch(() => context.input.url.toString()),
|
||||
adapter.checkLogin?.(context).catch(() => ({
|
||||
provider: adapter.name,
|
||||
state: "unknown" as const,
|
||||
})),
|
||||
]);
|
||||
|
||||
return {
|
||||
url,
|
||||
hasGate: Boolean(gate),
|
||||
loginState: login?.state ?? "unavailable",
|
||||
sessionReady: adapter.name === "x" ? await isXSessionReady(context).catch(() => false) : true,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForForceResume(
|
||||
adapter: Adapter,
|
||||
context: AdapterContext,
|
||||
options: ConvertCommandOptions,
|
||||
): Promise<void> {
|
||||
if (context.interactive) {
|
||||
await context.browser.bringToFront().catch(() => {});
|
||||
}
|
||||
|
||||
const prompt =
|
||||
"Chrome is ready. Complete any manual login or verification. Extraction will continue automatically after it detects progress, or press Enter to continue immediately.";
|
||||
context.log.info(prompt);
|
||||
|
||||
const rl = createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stderr,
|
||||
});
|
||||
|
||||
let manualContinue = false;
|
||||
let closed = false;
|
||||
const closeReadline = (): void => {
|
||||
if (!closed) {
|
||||
closed = true;
|
||||
rl.close();
|
||||
}
|
||||
};
|
||||
|
||||
rl.once("line", () => {
|
||||
manualContinue = true;
|
||||
closeReadline();
|
||||
});
|
||||
|
||||
const initial = await captureForceWaitSnapshot(adapter, context);
|
||||
const startedAt = Date.now();
|
||||
|
||||
try {
|
||||
while (Date.now() - startedAt < options.interactionTimeoutMs) {
|
||||
if (manualContinue) {
|
||||
return;
|
||||
}
|
||||
|
||||
const current = await captureForceWaitSnapshot(adapter, context);
|
||||
if (shouldAutoContinueForceWait(initial, current)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await sleep(options.interactionPollIntervalMs);
|
||||
}
|
||||
} finally {
|
||||
closeReadline();
|
||||
}
|
||||
|
||||
throw new Error("Timed out waiting for force-mode interaction to complete");
|
||||
}
|
||||
|
||||
async function waitForInteraction(
|
||||
adapter: Adapter,
|
||||
context: AdapterContext,
|
||||
interaction: WaitForInteractionRequest,
|
||||
options: ConvertCommandOptions,
|
||||
): Promise<AdapterLoginInfo> {
|
||||
const timeoutMs = interaction.timeoutMs ?? options.interactionTimeoutMs;
|
||||
const pollIntervalMs = interaction.pollIntervalMs ?? options.interactionPollIntervalMs;
|
||||
if (context.interactive) {
|
||||
await context.browser.bringToFront().catch(() => {});
|
||||
}
|
||||
context.log.info(interaction.prompt);
|
||||
|
||||
const startedAt = Date.now();
|
||||
let lastLogin: AdapterLoginInfo | null = null;
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (interaction.kind === "login" && adapter.checkLogin) {
|
||||
lastLogin = await adapter.checkLogin(context);
|
||||
if (lastLogin.state === "logged_in" && await isInteractionSessionReady(context, interaction)) {
|
||||
return lastLogin;
|
||||
}
|
||||
}
|
||||
|
||||
const gate = await detectInteractionGate(context.browser);
|
||||
if (!gate) {
|
||||
if (interaction.kind !== "login") {
|
||||
return lastLogin ?? {
|
||||
provider: interaction.provider,
|
||||
state: "unknown",
|
||||
reason: `${interaction.provider} challenge cleared`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!adapter.checkLogin) {
|
||||
return {
|
||||
provider: interaction.provider,
|
||||
state: "unknown",
|
||||
};
|
||||
}
|
||||
|
||||
lastLogin = await adapter.checkLogin(context);
|
||||
if (lastLogin.state !== "logged_out" && await isInteractionSessionReady(context, interaction)) {
|
||||
return lastLogin;
|
||||
}
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
|
||||
const reason = lastLogin?.reason ? ` (${lastLogin.reason})` : "";
|
||||
throw new Error(`Timed out waiting for ${interaction.provider} interaction${reason}`);
|
||||
}
|
||||
|
||||
export function formatOutputContent(
|
||||
format: OutputFormat,
|
||||
payload: SuccessfulConvertOutput | InteractionRequiredOutput,
|
||||
): string {
|
||||
if (format === "json") {
|
||||
return JSON.stringify(payload, null, 2);
|
||||
}
|
||||
|
||||
if (payload.status !== "ok") {
|
||||
throw new Error("Markdown output is only available for successful extraction results");
|
||||
}
|
||||
|
||||
return payload.markdown;
|
||||
}
|
||||
|
||||
function printOutput(content: string): void {
|
||||
process.stdout.write(content);
|
||||
if (!content.endsWith("\n")) {
|
||||
process.stdout.write("\n");
|
||||
}
|
||||
}
|
||||
|
||||
export async function runConvertCommand(options: ConvertCommandOptions): Promise<void> {
|
||||
if (!options.url) {
|
||||
throw new Error("URL is required");
|
||||
}
|
||||
if (options.downloadMedia && !options.output) {
|
||||
throw new Error("--download-media requires --output so media paths can be rewritten relative to the saved output file");
|
||||
}
|
||||
|
||||
const url = normalizeUrl(options.url);
|
||||
let runtime = await openRuntime(options, options.waitMode !== "none", Boolean(options.debugDir));
|
||||
const logger = createLogger(Boolean(options.debugDir));
|
||||
let didLogin = false;
|
||||
let adapter: Adapter | null = null;
|
||||
let context: AdapterContext | null = null;
|
||||
|
||||
try {
|
||||
adapter = resolveAdapter({ url }, options.adapter);
|
||||
context = {
|
||||
input: { url },
|
||||
browser: runtime.browser,
|
||||
network: runtime.network,
|
||||
cdp: runtime.cdp,
|
||||
log: logger,
|
||||
outputFormat: options.format,
|
||||
timeoutMs: options.timeoutMs,
|
||||
interactive: runtime.interactive,
|
||||
downloadMedia: options.downloadMedia,
|
||||
};
|
||||
|
||||
if (adapter.restoreCookies) {
|
||||
const restored = await adapter.restoreCookies(context, runtime.chrome.profileDir).catch(() => false);
|
||||
if (restored) logger.info(`Restored ${adapter.name} session cookies from sidecar.`);
|
||||
}
|
||||
|
||||
if (options.waitMode === "interaction" && adapter.checkLogin) {
|
||||
await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
|
||||
const preLogin = await adapter.checkLogin(context);
|
||||
if (preLogin.state !== "logged_in") {
|
||||
didLogin = true;
|
||||
await waitForInteraction(adapter, context, {
|
||||
type: "wait_for_interaction",
|
||||
kind: "login",
|
||||
provider: preLogin.provider ?? adapter.name,
|
||||
prompt: `Please sign in to ${adapter.name === "x" ? "X" : adapter.name} in the opened Chrome window. Extraction will continue automatically once login is detected.`,
|
||||
reason: preLogin.reason ?? `Not logged in to ${adapter.name}`,
|
||||
requiresVisibleBrowser: true,
|
||||
}, options);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.waitMode === "force") {
|
||||
await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
|
||||
await waitForForceResume(adapter, context, options);
|
||||
}
|
||||
|
||||
let result = await adapter.process(context);
|
||||
|
||||
if (result.status === "no_document") {
|
||||
const interaction = await detectInteractionGate(context.browser);
|
||||
if (interaction) {
|
||||
result = {
|
||||
status: "needs_interaction",
|
||||
interaction,
|
||||
login: result.login,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
while (result.status === "needs_interaction") {
|
||||
if (options.waitMode === "none") {
|
||||
if (options.format === "json") {
|
||||
printOutput(
|
||||
formatOutputContent(options.format, {
|
||||
adapter: adapter.name,
|
||||
status: result.status,
|
||||
login: result.login,
|
||||
interaction: result.interaction,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`${adapter.name} requires manual interaction. Re-run with --wait-for interaction to continue after completing it.`);
|
||||
}
|
||||
|
||||
if (result.interaction.requiresVisibleBrowser !== false) {
|
||||
runtime = await reopenInteractiveRuntime(runtime, options, Boolean(options.debugDir));
|
||||
}
|
||||
|
||||
context = {
|
||||
input: { url },
|
||||
browser: runtime.browser,
|
||||
network: runtime.network,
|
||||
cdp: runtime.cdp,
|
||||
log: logger,
|
||||
outputFormat: options.format,
|
||||
timeoutMs: options.timeoutMs,
|
||||
interactive: runtime.interactive,
|
||||
downloadMedia: options.downloadMedia,
|
||||
};
|
||||
|
||||
await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
|
||||
if (result.interaction.kind === "login") {
|
||||
didLogin = true;
|
||||
}
|
||||
await waitForInteraction(adapter, context, result.interaction, options);
|
||||
result = await adapter.process(context);
|
||||
|
||||
if (result.status === "no_document") {
|
||||
const interaction = await detectInteractionGate(context.browser);
|
||||
if (interaction) {
|
||||
result = {
|
||||
status: "needs_interaction",
|
||||
interaction,
|
||||
login: result.login,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let document: ExtractedDocument | null = result.status === "ok" ? result.document : null;
|
||||
let media: MediaAsset[] = result.status === "ok" ? (result.media ?? []) : [];
|
||||
let login = result.login;
|
||||
let mediaAdapter = adapter;
|
||||
|
||||
if (!document && adapter.name !== genericAdapter.name && result.status === "no_document") {
|
||||
logger.info(`Adapter ${adapter.name} returned no structured document; falling back to generic extraction`);
|
||||
const fallback = await genericAdapter.process(context);
|
||||
if (fallback.status === "ok") {
|
||||
document = fallback.document;
|
||||
media = fallback.media ?? [];
|
||||
mediaAdapter = genericAdapter;
|
||||
}
|
||||
}
|
||||
|
||||
if (!document) {
|
||||
throw new Error("Failed to extract a document from the target URL");
|
||||
}
|
||||
|
||||
document.requestedUrl ??= url.toString();
|
||||
|
||||
let markdown = renderMarkdown(document);
|
||||
let downloadResult:
|
||||
| Awaited<ReturnType<typeof downloadMediaAssets>>
|
||||
| null = null;
|
||||
|
||||
if (options.downloadMedia && options.output) {
|
||||
downloadResult = mediaAdapter.downloadMedia
|
||||
? await mediaAdapter.downloadMedia({
|
||||
media,
|
||||
outputPath: options.output,
|
||||
mediaDir: options.mediaDir,
|
||||
log: logger,
|
||||
})
|
||||
: await downloadMediaAssets({
|
||||
media,
|
||||
outputPath: options.output,
|
||||
mediaDir: options.mediaDir,
|
||||
log: logger,
|
||||
});
|
||||
|
||||
markdown = rewriteMarkdownMediaLinks(markdown, downloadResult.replacements);
|
||||
if (downloadResult.downloadedImages > 0 || downloadResult.downloadedVideos > 0) {
|
||||
logger.info(
|
||||
`Downloaded ${downloadResult.downloadedImages} images and ${downloadResult.downloadedVideos} videos`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.output) {
|
||||
await writeOutput(
|
||||
options.output,
|
||||
formatOutputContent(options.format, {
|
||||
adapter: document.adapter ?? adapter.name,
|
||||
status: "ok",
|
||||
login,
|
||||
media,
|
||||
downloads: downloadResult,
|
||||
document,
|
||||
markdown,
|
||||
}),
|
||||
);
|
||||
logger.info(`Saved ${options.format} to ${options.output}`);
|
||||
}
|
||||
|
||||
if (options.debugDir) {
|
||||
await writeDebugArtifacts(options.debugDir, document, markdown, runtime.browser, runtime.network);
|
||||
logger.info(`Wrote debug artifacts to ${options.debugDir}`);
|
||||
}
|
||||
|
||||
if (options.format === "json") {
|
||||
printOutput(
|
||||
formatOutputContent(options.format, {
|
||||
adapter: document.adapter ?? adapter.name,
|
||||
status: "ok",
|
||||
login,
|
||||
media,
|
||||
downloads: downloadResult,
|
||||
document,
|
||||
markdown,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
printOutput(markdown);
|
||||
} finally {
|
||||
if (adapter?.exportCookies && context) {
|
||||
await adapter.exportCookies(context, runtime.chrome.profileDir).catch(() => {});
|
||||
}
|
||||
await closeRuntime(runtime);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
export type ContentBlock =
|
||||
| {
|
||||
type: "paragraph";
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "heading";
|
||||
depth: number;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "list";
|
||||
ordered: boolean;
|
||||
items: string[];
|
||||
}
|
||||
| {
|
||||
type: "quote";
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "code";
|
||||
code: string;
|
||||
language?: string;
|
||||
}
|
||||
| {
|
||||
type: "image";
|
||||
url: string;
|
||||
alt?: string;
|
||||
}
|
||||
| {
|
||||
type: "html";
|
||||
html: string;
|
||||
}
|
||||
| {
|
||||
type: "markdown";
|
||||
markdown: string;
|
||||
};
|
||||
|
||||
export interface ExtractedDocument {
|
||||
url: string;
|
||||
requestedUrl?: string;
|
||||
canonicalUrl?: string;
|
||||
title?: string;
|
||||
author?: string;
|
||||
siteName?: string;
|
||||
publishedAt?: string;
|
||||
summary?: string;
|
||||
content: ContentBlock[];
|
||||
metadata?: Record<string, unknown>;
|
||||
adapter?: string;
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
|
||||
export interface CleanHtmlOptions {
|
||||
removeAds?: boolean;
|
||||
removeBase64Images?: boolean;
|
||||
onlyMainContent?: boolean;
|
||||
includeSelectors?: string[];
|
||||
excludeSelectors?: string[];
|
||||
}
|
||||
|
||||
const ALWAYS_REMOVE_SELECTORS = [
|
||||
"script",
|
||||
"style",
|
||||
"noscript",
|
||||
"link[rel='stylesheet']",
|
||||
"[hidden]",
|
||||
"[aria-hidden='true']",
|
||||
"[style*='display: none']",
|
||||
"[style*='display:none']",
|
||||
"[style*='visibility: hidden']",
|
||||
"[style*='visibility:hidden']",
|
||||
"svg[aria-hidden='true']",
|
||||
"svg.icon",
|
||||
"svg[class*='icon']",
|
||||
"template",
|
||||
"meta",
|
||||
"iframe",
|
||||
"canvas",
|
||||
"object",
|
||||
"embed",
|
||||
"form",
|
||||
"input",
|
||||
"select",
|
||||
"textarea",
|
||||
"button",
|
||||
];
|
||||
|
||||
const OVERLAY_SELECTORS = [
|
||||
"[class*='modal']",
|
||||
"[class*='popup']",
|
||||
"[class*='overlay']",
|
||||
"[class*='dialog']",
|
||||
"[role='dialog']",
|
||||
"[role='alertdialog']",
|
||||
"[class*='cookie']",
|
||||
"[class*='consent']",
|
||||
"[class*='gdpr']",
|
||||
"[class*='privacy-banner']",
|
||||
"[class*='notification-bar']",
|
||||
"[id*='cookie']",
|
||||
"[id*='consent']",
|
||||
"[id*='gdpr']",
|
||||
"[style*='position: fixed']",
|
||||
"[style*='position:fixed']",
|
||||
"[style*='position: sticky']",
|
||||
"[style*='position:sticky']",
|
||||
];
|
||||
|
||||
const NAVIGATION_SELECTORS = [
|
||||
"header",
|
||||
"footer",
|
||||
"nav",
|
||||
"aside",
|
||||
".header",
|
||||
".top",
|
||||
".navbar",
|
||||
"#header",
|
||||
".footer",
|
||||
".bottom",
|
||||
"#footer",
|
||||
".sidebar",
|
||||
".side",
|
||||
".aside",
|
||||
"#sidebar",
|
||||
".modal",
|
||||
".popup",
|
||||
"#modal",
|
||||
".overlay",
|
||||
".ad",
|
||||
".ads",
|
||||
".advert",
|
||||
"#ad",
|
||||
".lang-selector",
|
||||
".language",
|
||||
"#language-selector",
|
||||
".social",
|
||||
".social-media",
|
||||
".social-links",
|
||||
"#social",
|
||||
".menu",
|
||||
".navigation",
|
||||
"#nav",
|
||||
".breadcrumbs",
|
||||
"#breadcrumbs",
|
||||
".share",
|
||||
"#share",
|
||||
".widget",
|
||||
"#widget",
|
||||
".cookie",
|
||||
"#cookie",
|
||||
];
|
||||
|
||||
const FORCE_INCLUDE_SELECTORS = [
|
||||
"#main",
|
||||
"#content",
|
||||
"#main-content",
|
||||
"#article",
|
||||
"#post",
|
||||
"#page-content",
|
||||
"main",
|
||||
"article",
|
||||
"[role='main']",
|
||||
".main-content",
|
||||
".content",
|
||||
".post-content",
|
||||
".article-content",
|
||||
".entry-content",
|
||||
".page-content",
|
||||
".article-body",
|
||||
".post-body",
|
||||
".story-content",
|
||||
".blog-content",
|
||||
];
|
||||
|
||||
const AD_SELECTORS = [
|
||||
"ins.adsbygoogle",
|
||||
".google-ad",
|
||||
".adsense",
|
||||
"[data-ad]",
|
||||
"[data-ads]",
|
||||
"[data-ad-slot]",
|
||||
"[data-ad-client]",
|
||||
".ad-container",
|
||||
".ad-wrapper",
|
||||
".advertisement",
|
||||
".sponsored-content",
|
||||
"img[width='1'][height='1']",
|
||||
"img[src*='pixel']",
|
||||
"img[src*='tracking']",
|
||||
"img[src*='analytics']",
|
||||
];
|
||||
|
||||
function getLinkDensity(element: Element): number {
|
||||
const text = element.textContent || "";
|
||||
const textLength = text.trim().length;
|
||||
if (textLength === 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
let linkLength = 0;
|
||||
element.querySelectorAll("a").forEach((link) => {
|
||||
linkLength += (link.textContent || "").trim().length;
|
||||
});
|
||||
|
||||
return linkLength / textLength;
|
||||
}
|
||||
|
||||
function getContentScore(element: Element): number {
|
||||
let score = 0;
|
||||
const text = element.textContent || "";
|
||||
const textLength = text.trim().length;
|
||||
|
||||
score += Math.min(textLength / 100, 50);
|
||||
score += element.querySelectorAll("p").length * 3;
|
||||
score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2;
|
||||
score += element.querySelectorAll("img").length;
|
||||
|
||||
score -= element.querySelectorAll("a").length * 0.5;
|
||||
score -= element.querySelectorAll("li").length * 0.2;
|
||||
|
||||
const linkDensity = getLinkDensity(element);
|
||||
if (linkDensity > 0.5) {
|
||||
score -= 30;
|
||||
} else if (linkDensity > 0.3) {
|
||||
score -= 15;
|
||||
}
|
||||
|
||||
const className = typeof element.className === "string" ? element.className : "";
|
||||
const classAndId = `${className} ${element.id || ""}`;
|
||||
if (/article|content|post|body|main|entry/i.test(classAndId)) {
|
||||
score += 25;
|
||||
}
|
||||
if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) {
|
||||
score -= 25;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
function looksLikeNavigation(element: Element): boolean {
|
||||
const linkDensity = getLinkDensity(element);
|
||||
if (linkDensity > 0.5) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const listItems = element.querySelectorAll("li");
|
||||
const links = element.querySelectorAll("a");
|
||||
return listItems.length > 5 && links.length > listItems.length * 0.8;
|
||||
}
|
||||
|
||||
function removeElements(document: Document, selectors: string[]): void {
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
document.querySelectorAll(selector).forEach((element) => element.remove());
|
||||
} catch {
|
||||
// Ignore unsupported selectors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function removeWithProtection(
|
||||
document: Document,
|
||||
selectorsToRemove: string[],
|
||||
protectedSelectors: string[],
|
||||
): void {
|
||||
for (const selector of selectorsToRemove) {
|
||||
try {
|
||||
document.querySelectorAll(selector).forEach((element) => {
|
||||
const isProtected = protectedSelectors.some((protectedSelector) => {
|
||||
try {
|
||||
return element.matches(protectedSelector);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
if (isProtected) {
|
||||
return;
|
||||
}
|
||||
|
||||
const containsProtected = protectedSelectors.some((protectedSelector) => {
|
||||
try {
|
||||
return element.querySelector(protectedSelector) !== null;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
if (containsProtected) {
|
||||
return;
|
||||
}
|
||||
|
||||
element.remove();
|
||||
});
|
||||
} catch {
|
||||
// Ignore unsupported selectors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isValidContent(element: Element | null): element is Element {
|
||||
if (!element) {
|
||||
return false;
|
||||
}
|
||||
const text = element.textContent || "";
|
||||
if (text.trim().length < 100) {
|
||||
return false;
|
||||
}
|
||||
return !looksLikeNavigation(element);
|
||||
}
|
||||
|
||||
function findMainContent(document: Document): Element | null {
|
||||
const main = document.querySelector("main");
|
||||
if (isValidContent(main) && getLinkDensity(main) < 0.4) {
|
||||
return main;
|
||||
}
|
||||
|
||||
const roleMain = document.querySelector('[role="main"]');
|
||||
if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) {
|
||||
return roleMain;
|
||||
}
|
||||
|
||||
const articles = document.querySelectorAll("article");
|
||||
if (articles.length === 1 && isValidContent(articles[0] ?? null)) {
|
||||
return articles[0] ?? null;
|
||||
}
|
||||
|
||||
const contentSelectors = [
|
||||
"#content",
|
||||
"#main-content",
|
||||
"#main",
|
||||
".content",
|
||||
".main-content",
|
||||
".post-content",
|
||||
".article-content",
|
||||
".entry-content",
|
||||
".page-content",
|
||||
".article-body",
|
||||
".post-body",
|
||||
".story-content",
|
||||
".blog-content",
|
||||
];
|
||||
|
||||
for (const selector of contentSelectors) {
|
||||
try {
|
||||
const element = document.querySelector(selector);
|
||||
if (isValidContent(element) && getLinkDensity(element) < 0.4) {
|
||||
return element;
|
||||
}
|
||||
} catch {
|
||||
// Ignore invalid selectors.
|
||||
}
|
||||
}
|
||||
|
||||
const candidates: Array<{ element: Element; score: number }> = [];
|
||||
document.querySelectorAll("div, section, article").forEach((element) => {
|
||||
const text = element.textContent || "";
|
||||
if (text.trim().length < 200) {
|
||||
return;
|
||||
}
|
||||
|
||||
const score = getContentScore(element);
|
||||
if (score > 0) {
|
||||
candidates.push({ element, score });
|
||||
}
|
||||
});
|
||||
|
||||
candidates.sort((left, right) => right.score - left.score);
|
||||
if ((candidates[0]?.score ?? 0) > 20) {
|
||||
return candidates[0]?.element ?? null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function removeBase64ImagesFromDocument(document: Document): void {
|
||||
document.querySelectorAll("img[src^='data:']").forEach((element) => element.remove());
|
||||
|
||||
document.querySelectorAll("[style*='data:image']").forEach((element) => {
|
||||
const style = element.getAttribute("style");
|
||||
if (!style) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cleanedStyle = style.replace(
|
||||
/background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi,
|
||||
"",
|
||||
);
|
||||
|
||||
if (cleanedStyle.trim()) {
|
||||
element.setAttribute("style", cleanedStyle);
|
||||
} else {
|
||||
element.removeAttribute("style");
|
||||
}
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll("source[src^='data:'], source[srcset*='data:']")
|
||||
.forEach((element) => element.remove());
|
||||
}
|
||||
|
||||
function makeAbsoluteUrl(value: string, baseUrl: string): string | null {
|
||||
try {
|
||||
return new URL(value, baseUrl).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function convertRelativeUrls(document: Document, baseUrl: string): void {
|
||||
document.querySelectorAll("[src]").forEach((element) => {
|
||||
const src = element.getAttribute("src");
|
||||
if (!src || src.startsWith("http") || src.startsWith("//") || src.startsWith("data:")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const absolute = makeAbsoluteUrl(src, baseUrl);
|
||||
if (absolute) {
|
||||
element.setAttribute("src", absolute);
|
||||
}
|
||||
});
|
||||
|
||||
document.querySelectorAll("[href]").forEach((element) => {
|
||||
const href = element.getAttribute("href");
|
||||
if (
|
||||
!href ||
|
||||
href.startsWith("http") ||
|
||||
href.startsWith("//") ||
|
||||
href.startsWith("#") ||
|
||||
href.startsWith("mailto:") ||
|
||||
href.startsWith("tel:") ||
|
||||
href.startsWith("javascript:")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const absolute = makeAbsoluteUrl(href, baseUrl);
|
||||
if (absolute) {
|
||||
element.setAttribute("href", absolute);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function removeComments(document: Document): void {
|
||||
const walker = document.createTreeWalker(document, document.defaultView?.NodeFilter.SHOW_COMMENT ?? 128);
|
||||
const comments: Comment[] = [];
|
||||
while (walker.nextNode()) {
|
||||
comments.push(walker.currentNode as Comment);
|
||||
}
|
||||
comments.forEach((comment) => comment.parentNode?.removeChild(comment));
|
||||
}
|
||||
|
||||
export function cleanHtml(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
options: CleanHtmlOptions = {},
|
||||
): string {
|
||||
const {
|
||||
removeAds = true,
|
||||
removeBase64Images = true,
|
||||
onlyMainContent = true,
|
||||
includeSelectors,
|
||||
excludeSelectors,
|
||||
} = options;
|
||||
|
||||
const dom = new JSDOM(html, { url: baseUrl });
|
||||
const { document } = dom.window;
|
||||
|
||||
removeElements(document, ALWAYS_REMOVE_SELECTORS);
|
||||
removeElements(document, OVERLAY_SELECTORS);
|
||||
|
||||
if (removeAds) {
|
||||
removeElements(document, AD_SELECTORS);
|
||||
}
|
||||
|
||||
if (excludeSelectors?.length) {
|
||||
removeElements(document, excludeSelectors);
|
||||
}
|
||||
|
||||
if (onlyMainContent) {
|
||||
removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS);
|
||||
|
||||
const mainContent = findMainContent(document);
|
||||
if (mainContent && document.body) {
|
||||
const clone = mainContent.cloneNode(true);
|
||||
document.body.innerHTML = "";
|
||||
document.body.appendChild(clone);
|
||||
}
|
||||
}
|
||||
|
||||
if (includeSelectors?.length && document.body) {
|
||||
const matchedElements: Element[] = [];
|
||||
for (const selector of includeSelectors) {
|
||||
try {
|
||||
document.querySelectorAll(selector).forEach((element) => {
|
||||
matchedElements.push(element.cloneNode(true) as Element);
|
||||
});
|
||||
} catch {
|
||||
// Ignore invalid selectors.
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedElements.length > 0) {
|
||||
document.body.innerHTML = "";
|
||||
matchedElements.forEach((element) => document.body?.appendChild(element));
|
||||
}
|
||||
}
|
||||
|
||||
if (removeBase64Images) {
|
||||
removeBase64ImagesFromDocument(document);
|
||||
}
|
||||
|
||||
removeComments(document);
|
||||
convertRelativeUrls(document, baseUrl);
|
||||
|
||||
return document.documentElement.outerHTML || html;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { JSDOM } from "jsdom";
|
||||
import type { ExtractedDocument } from "./document";
|
||||
|
||||
function getMetaContent(document: Document, selectors: string[]): string | undefined {
|
||||
for (const selector of selectors) {
|
||||
const value = document.querySelector(selector)?.getAttribute("content")?.trim();
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function extractDocumentFromHtml(input: {
|
||||
url: string;
|
||||
html: string;
|
||||
adapter?: string;
|
||||
}): ExtractedDocument {
|
||||
const dom = new JSDOM(input.html, { url: input.url });
|
||||
const document = dom.window.document;
|
||||
|
||||
const canonicalUrl =
|
||||
document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
|
||||
getMetaContent(document, ['meta[property="og:url"]']);
|
||||
|
||||
const siteName = getMetaContent(document, [
|
||||
'meta[property="og:site_name"]',
|
||||
'meta[name="application-name"]',
|
||||
]);
|
||||
|
||||
const metadataAuthor = getMetaContent(document, [
|
||||
'meta[name="author"]',
|
||||
'meta[property="article:author"]',
|
||||
'meta[name="twitter:creator"]',
|
||||
]);
|
||||
|
||||
const publishedAt = getMetaContent(document, [
|
||||
'meta[property="article:published_time"]',
|
||||
'meta[name="pubdate"]',
|
||||
'meta[name="date"]',
|
||||
'meta[itemprop="datePublished"]',
|
||||
]);
|
||||
|
||||
const article = new Readability(document).parse();
|
||||
const title =
|
||||
article?.title?.trim() ||
|
||||
getMetaContent(document, ['meta[property="og:title"]']) ||
|
||||
document.title.trim() ||
|
||||
undefined;
|
||||
|
||||
const summary =
|
||||
article?.excerpt?.trim() ||
|
||||
getMetaContent(document, [
|
||||
'meta[name="description"]',
|
||||
'meta[property="og:description"]',
|
||||
'meta[name="twitter:description"]',
|
||||
]);
|
||||
|
||||
const contentHtml =
|
||||
article?.content?.trim() ||
|
||||
document.querySelector("main")?.innerHTML?.trim() ||
|
||||
document.body?.innerHTML?.trim() ||
|
||||
"";
|
||||
|
||||
const author = article?.byline?.trim() || metadataAuthor;
|
||||
|
||||
return {
|
||||
url: input.url,
|
||||
canonicalUrl,
|
||||
title,
|
||||
author,
|
||||
siteName,
|
||||
publishedAt,
|
||||
summary,
|
||||
adapter: input.adapter ?? "generic",
|
||||
metadata: {
|
||||
language: document.documentElement.lang || undefined,
|
||||
},
|
||||
content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,758 @@
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { Defuddle } from "defuddle/node";
|
||||
import { JSDOM, VirtualConsole } from "jsdom";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
import { collectMediaFromMarkdown } from "../media/markdown-media";
|
||||
import type { MediaAsset } from "../media/types";
|
||||
import { cleanHtml } from "./html-cleaner";
|
||||
|
||||
export interface HtmlConversionMetadata {
|
||||
url: string;
|
||||
canonicalUrl?: string;
|
||||
siteName?: string;
|
||||
title?: string;
|
||||
summary?: string;
|
||||
author?: string;
|
||||
publishedAt?: string;
|
||||
coverImage?: string;
|
||||
language?: string;
|
||||
capturedAt: string;
|
||||
}
|
||||
|
||||
export interface ConvertHtmlToMarkdownOptions {
|
||||
enableRemoteMarkdownFallback?: boolean;
|
||||
preserveBase64Images?: boolean;
|
||||
}
|
||||
|
||||
export interface HtmlToMarkdownResult {
|
||||
metadata: HtmlConversionMetadata;
|
||||
markdown: string;
|
||||
rawHtml: string;
|
||||
cleanedHtml: string;
|
||||
media: MediaAsset[];
|
||||
conversionMethod: string;
|
||||
fallbackReason?: string;
|
||||
}
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
const MIN_CONTENT_LENGTH = 120;
|
||||
const DEFUDDLE_API_ORIGIN = "https://defuddle.md";
|
||||
const LOCAL_FALLBACK_SCORE_DELTA = 120;
|
||||
const REMOTE_FALLBACK_SCORE_DELTA = 20;
|
||||
const LOW_QUALITY_MARKERS = [
|
||||
/Join The Conversation/i,
|
||||
/One Community\. Many Voices/i,
|
||||
/Read our community guidelines/i,
|
||||
/Create a free account to share your thoughts/i,
|
||||
/Become a Forbes Member/i,
|
||||
/Subscribe to trusted journalism/i,
|
||||
/\bComments\b/i,
|
||||
];
|
||||
|
||||
const ARTICLE_TYPES = new Set([
|
||||
"Article",
|
||||
"NewsArticle",
|
||||
"BlogPosting",
|
||||
"WebPage",
|
||||
"ReportageNewsArticle",
|
||||
]);
|
||||
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: "atx",
|
||||
bulletListMarker: "-",
|
||||
codeBlockStyle: "fenced",
|
||||
}) as TurndownService & {
|
||||
remove(selectors: string[]): void;
|
||||
addRule(
|
||||
key: string,
|
||||
rule: {
|
||||
filter: string | ((node: Node) => boolean);
|
||||
replacement: (content: string) => string;
|
||||
},
|
||||
): void;
|
||||
};
|
||||
|
||||
turndown.use(gfm);
|
||||
turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
|
||||
turndown.addRule("collapseFigure", {
|
||||
filter: "figure",
|
||||
replacement(content: string) {
|
||||
return `\n\n${content.trim()}\n\n`;
|
||||
},
|
||||
});
|
||||
turndown.addRule("dropInvisibleAnchors", {
|
||||
filter(node: Node) {
|
||||
return (
|
||||
node.nodeName === "A" &&
|
||||
!(node as Element).textContent?.trim() &&
|
||||
!(node as Element).querySelector("img, video, picture, source")
|
||||
);
|
||||
},
|
||||
replacement() {
|
||||
return "";
|
||||
},
|
||||
});
|
||||
|
||||
function pickString(...values: unknown[]): string | undefined {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeMarkdown(markdown: string): string {
|
||||
return markdown
|
||||
.replace(/\r\n/g, "\n")
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function stripWrappingQuotes(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (
|
||||
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
||||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
||||
) {
|
||||
return trimmed.slice(1, -1).trim();
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function stripMarkdownFrontmatter(markdown: string): string {
|
||||
return markdown.replace(/^\uFEFF?---\n[\s\S]*?\n---(?:\n|$)/, "").trim();
|
||||
}
|
||||
|
||||
function cleanMarkdownTitle(value: string): string | undefined {
|
||||
const cleaned = stripWrappingQuotes(
|
||||
value
|
||||
.replace(/\s+#+\s*$/, "")
|
||||
.replace(/!\[[^\]]*\]\([^)]+\)/g, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.replace(/[*_`~]/g, "")
|
||||
.trim(),
|
||||
);
|
||||
|
||||
return cleaned || undefined;
|
||||
}
|
||||
|
||||
export function extractTitleFromMarkdownDocument(markdown: string): string | undefined {
|
||||
const normalized = markdown.replace(/\r\n/g, "\n").trim();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const frontmatterMatch = normalized.match(/^\uFEFF?---\n([\s\S]*?)\n---(?:\n|$)/);
|
||||
if (frontmatterMatch) {
|
||||
for (const line of frontmatterMatch[1].split("\n")) {
|
||||
const match = line.match(/^title:\s*(.+?)\s*$/i);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const title = cleanMarkdownTitle(match[1]);
|
||||
if (title) {
|
||||
return title;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const body = stripMarkdownFrontmatter(normalized);
|
||||
const headingMatch = body.match(/^#{1,6}\s+(.+)$/m);
|
||||
if (!headingMatch) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return cleanMarkdownTitle(headingMatch[1]);
|
||||
}
|
||||
|
||||
function trimKnownBoilerplate(markdown: string): string {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
const lines = normalized.split("\n");
|
||||
|
||||
while (lines.length > 0) {
|
||||
const lastLine = lines[lines.length - 1]?.trim();
|
||||
if (!lastLine) {
|
||||
lines.pop();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^继续滑动看下一个$/.test(lastLine) || /^轻触阅读原文$/.test(lastLine)) {
|
||||
lines.pop();
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return normalizeMarkdown(lines.join("\n"));
|
||||
}
|
||||
|
||||
function buildDefuddleApiUrl(targetUrl: string): string {
|
||||
return `${DEFUDDLE_API_ORIGIN}/${encodeURIComponent(targetUrl)}`;
|
||||
}
|
||||
|
||||
async function fetchDefuddleApiMarkdown(
|
||||
targetUrl: string,
|
||||
): Promise<{ markdown: string; title?: string }> {
|
||||
const response = await fetch(buildDefuddleApiUrl(targetUrl), {
|
||||
headers: {
|
||||
accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
|
||||
},
|
||||
redirect: "follow",
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const rawMarkdown = (await response.text()).replace(/\r\n/g, "\n").trim();
|
||||
if (!rawMarkdown) {
|
||||
throw new Error("defuddle.md returned empty markdown");
|
||||
}
|
||||
|
||||
const title = extractTitleFromMarkdownDocument(rawMarkdown);
|
||||
const markdown = trimKnownBoilerplate(stripMarkdownFrontmatter(rawMarkdown));
|
||||
if (!markdown) {
|
||||
throw new Error("defuddle.md returned empty markdown");
|
||||
}
|
||||
|
||||
return {
|
||||
markdown,
|
||||
title,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeHtmlFragment(html: string): string {
|
||||
const dom = new JSDOM(`<div id="__root">${html}</div>`);
|
||||
const root = dom.window.document.querySelector("#__root");
|
||||
if (!root) {
|
||||
return html;
|
||||
}
|
||||
|
||||
for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
|
||||
root.querySelectorAll(selector).forEach((element) => element.remove());
|
||||
}
|
||||
|
||||
return root.innerHTML;
|
||||
}
|
||||
|
||||
function extractTextFromHtml(html: string): string {
|
||||
const dom = new JSDOM(`<!doctype html><html><body>${html}</body></html>`);
|
||||
const { document } = dom.window;
|
||||
for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
|
||||
document.querySelectorAll(selector).forEach((element) => element.remove());
|
||||
}
|
||||
return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
|
||||
}
|
||||
|
||||
function getMetaContent(document: Document, names: string[]): string | undefined {
|
||||
for (const name of names) {
|
||||
const element =
|
||||
document.querySelector(`meta[name="${name}"]`) ??
|
||||
document.querySelector(`meta[property="${name}"]`);
|
||||
const content = element?.getAttribute("content")?.trim();
|
||||
if (content) {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeLanguageTag(value: string | null | undefined): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim();
|
||||
if (!primary) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return primary.replace(/_/g, "-");
|
||||
}
|
||||
|
||||
function flattenJsonLdItems(data: unknown): JsonObject[] {
|
||||
if (!data || typeof data !== "object") {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
return data.flatMap(flattenJsonLdItems);
|
||||
}
|
||||
|
||||
const item = data as JsonObject;
|
||||
if (Array.isArray(item["@graph"])) {
|
||||
return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
|
||||
}
|
||||
|
||||
return [item];
|
||||
}
|
||||
|
||||
function parseJsonLdScripts(document: Document): JsonObject[] {
|
||||
const results: JsonObject[] = [];
|
||||
document.querySelectorAll("script[type='application/ld+json']").forEach((script) => {
|
||||
try {
|
||||
const data = JSON.parse(script.textContent ?? "");
|
||||
results.push(...flattenJsonLdItems(data));
|
||||
} catch {
|
||||
// Ignore malformed json-ld blocks.
|
||||
}
|
||||
});
|
||||
return results;
|
||||
}
|
||||
|
||||
function extractAuthorFromJsonLd(authorData: unknown): string | undefined {
|
||||
if (typeof authorData === "string") {
|
||||
return authorData.trim() || undefined;
|
||||
}
|
||||
|
||||
if (!authorData || typeof authorData !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (Array.isArray(authorData)) {
|
||||
return authorData
|
||||
.map((author) => extractAuthorFromJsonLd(author))
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join(", ") || undefined;
|
||||
}
|
||||
|
||||
const author = authorData as JsonObject;
|
||||
return pickString(author.name);
|
||||
}
|
||||
|
||||
function extractPrimaryJsonLdMeta(document: Document): Partial<HtmlConversionMetadata> {
|
||||
for (const item of parseJsonLdScripts(document)) {
|
||||
const type = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
|
||||
if (typeof type !== "string" || !ARTICLE_TYPES.has(type)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
title: pickString(item.headline, item.name),
|
||||
summary: pickString(item.description),
|
||||
author: extractAuthorFromJsonLd(item.author),
|
||||
publishedAt: pickString(item.datePublished, item.dateCreated),
|
||||
coverImage: pickString(
|
||||
item.image,
|
||||
(item.image as JsonObject | undefined)?.url,
|
||||
Array.isArray(item.image) ? item.image[0] : undefined,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
function extractPageMetadata(
|
||||
html: string,
|
||||
url: string,
|
||||
capturedAt: string,
|
||||
): HtmlConversionMetadata {
|
||||
const dom = new JSDOM(html, { url });
|
||||
const { document } = dom.window;
|
||||
const jsonLd = extractPrimaryJsonLdMeta(document);
|
||||
|
||||
return {
|
||||
url,
|
||||
canonicalUrl:
|
||||
document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
|
||||
getMetaContent(document, ["og:url"]),
|
||||
siteName: pickString(
|
||||
getMetaContent(document, ["og:site_name"]),
|
||||
document.querySelector('meta[name="application-name"]')?.getAttribute("content"),
|
||||
),
|
||||
title: pickString(
|
||||
getMetaContent(document, ["og:title", "twitter:title"]),
|
||||
jsonLd.title,
|
||||
document.querySelector("h1")?.textContent,
|
||||
document.title,
|
||||
),
|
||||
summary: pickString(
|
||||
getMetaContent(document, ["description", "og:description", "twitter:description"]),
|
||||
jsonLd.summary,
|
||||
),
|
||||
author: pickString(
|
||||
getMetaContent(document, ["author", "article:author", "twitter:creator"]),
|
||||
jsonLd.author,
|
||||
),
|
||||
publishedAt: pickString(
|
||||
document.querySelector("time[datetime]")?.getAttribute("datetime"),
|
||||
getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
|
||||
jsonLd.publishedAt,
|
||||
),
|
||||
coverImage: pickString(
|
||||
getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
|
||||
jsonLd.coverImage,
|
||||
),
|
||||
language: pickString(
|
||||
normalizeLanguageTag(document.documentElement.getAttribute("lang")),
|
||||
normalizeLanguageTag(
|
||||
pickString(
|
||||
getMetaContent(document, ["language", "content-language", "og:locale"]),
|
||||
document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content"),
|
||||
),
|
||||
),
|
||||
),
|
||||
capturedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function isMarkdownUsable(markdown: string, html: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const htmlTextLength = extractTextFromHtml(html).length;
|
||||
if (htmlTextLength < MIN_CONTENT_LENGTH) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.length >= 80) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
|
||||
}
|
||||
|
||||
function countMarkerHits(markdown: string, markers: RegExp[]): number {
|
||||
let hits = 0;
|
||||
for (const marker of markers) {
|
||||
if (marker.test(markdown)) {
|
||||
hits += 1;
|
||||
}
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
function countUsefulParagraphs(markdown: string): number {
|
||||
const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/);
|
||||
let count = 0;
|
||||
|
||||
for (const paragraph of paragraphs) {
|
||||
const trimmed = paragraph.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
if (/^#{1,6}\s+/.test(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) {
|
||||
continue;
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
function scoreMarkdownQuality(markdown: string): number {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length;
|
||||
const usefulParagraphs = countUsefulParagraphs(normalized);
|
||||
const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length;
|
||||
const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS);
|
||||
return Math.min(wordCount, 4000) + usefulParagraphs * 40 + headingCount * 10 - markerHits * 180;
|
||||
}
|
||||
|
||||
function shouldCompareWithFallback(markdown: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
return countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 || countUsefulParagraphs(normalized) < 6;
|
||||
}
|
||||
|
||||
function hasMeaningfulMarkdownStructure(markdown: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
countUsefulParagraphs(normalized) > 0 ||
|
||||
/^#{1,6}\s+/m.test(normalized) ||
|
||||
/^[-*]\s+/m.test(normalized) ||
|
||||
/^\d+\.\s+/m.test(normalized) ||
|
||||
/!\[[^\]]*\]\([^)]+\)/.test(normalized)
|
||||
);
|
||||
}
|
||||
|
||||
function shouldTryRemoteMarkdownFallback(
|
||||
markdown: string,
|
||||
html: string,
|
||||
options: ConvertHtmlToMarkdownOptions,
|
||||
): boolean {
|
||||
if (!options.enableRemoteMarkdownFallback) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !isMarkdownUsable(markdown, html) || shouldCompareWithFallback(markdown);
|
||||
}
|
||||
|
||||
function shouldPreferRemoteMarkdown(
|
||||
current: HtmlToMarkdownResult,
|
||||
remote: HtmlToMarkdownResult,
|
||||
html: string,
|
||||
): boolean {
|
||||
if (!isMarkdownUsable(current.markdown, html)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!hasMeaningfulMarkdownStructure(current.markdown) && hasMeaningfulMarkdownStructure(remote.markdown)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return scoreMarkdownQuality(remote.markdown) > scoreMarkdownQuality(current.markdown) + REMOTE_FALLBACK_SCORE_DELTA;
|
||||
}
|
||||
|
||||
function buildRemoteFallbackReason(current: HtmlToMarkdownResult, html: string): string {
|
||||
if (!isMarkdownUsable(current.markdown, html)) {
|
||||
return current.fallbackReason
|
||||
? `Used defuddle.md markdown fallback after local extraction failed: ${current.fallbackReason}`
|
||||
: "Used defuddle.md markdown fallback after local extraction returned empty or incomplete markdown";
|
||||
}
|
||||
|
||||
return "defuddle.md produced higher-quality markdown than local extraction";
|
||||
}
|
||||
|
||||
async function tryDefuddleConversion(
|
||||
html: string,
|
||||
url: string,
|
||||
baseMetadata: HtmlConversionMetadata,
|
||||
): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
|
||||
try {
|
||||
const virtualConsole = new VirtualConsole();
|
||||
virtualConsole.on("jsdomError", (error: Error & { type?: string }) => {
|
||||
if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) {
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
const dom = new JSDOM(html, { url, virtualConsole });
|
||||
const result = await Defuddle(dom, url, { markdown: true });
|
||||
const markdown = trimKnownBoilerplate(result.content || "");
|
||||
|
||||
if (!isMarkdownUsable(markdown, html)) {
|
||||
return { ok: false, reason: "Defuddle returned empty or incomplete markdown" };
|
||||
}
|
||||
|
||||
const metadata: HtmlConversionMetadata = {
|
||||
...baseMetadata,
|
||||
title: pickString(result.title, baseMetadata.title),
|
||||
summary: pickString(result.description, baseMetadata.summary),
|
||||
author: pickString(result.author, baseMetadata.author),
|
||||
publishedAt: pickString(result.published, baseMetadata.publishedAt),
|
||||
coverImage: pickString(result.image, baseMetadata.coverImage),
|
||||
language: pickString(result.language, baseMetadata.language),
|
||||
};
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
metadata,
|
||||
markdown,
|
||||
rawHtml: html,
|
||||
cleanedHtml: html,
|
||||
media: collectMediaFromMarkdown(markdown).concat(
|
||||
metadata.coverImage
|
||||
? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
|
||||
: [],
|
||||
),
|
||||
conversionMethod: "defuddle",
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function tryDefuddleApiConversion(
|
||||
html: string,
|
||||
url: string,
|
||||
baseMetadata: HtmlConversionMetadata,
|
||||
): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
|
||||
try {
|
||||
const result = await fetchDefuddleApiMarkdown(url);
|
||||
const markdown = result.markdown;
|
||||
|
||||
if (!isMarkdownUsable(markdown, html) && scoreMarkdownQuality(markdown) < 80) {
|
||||
return { ok: false, reason: "defuddle.md returned empty or incomplete markdown" };
|
||||
}
|
||||
|
||||
const metadata: HtmlConversionMetadata = {
|
||||
...baseMetadata,
|
||||
title: pickString(result.title, baseMetadata.title),
|
||||
};
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
metadata,
|
||||
markdown,
|
||||
rawHtml: html,
|
||||
cleanedHtml: html,
|
||||
media: collectMediaFromMarkdown(markdown).concat(
|
||||
metadata.coverImage
|
||||
? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
|
||||
: [],
|
||||
),
|
||||
conversionMethod: "defuddle-api",
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function convertHtmlFragmentToMarkdown(html: string): string {
|
||||
if (!html.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
return turndown.turndown(sanitizeHtmlFragment(html));
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackPlainText(html: string): string {
|
||||
return trimKnownBoilerplate(extractTextFromHtml(html));
|
||||
}
|
||||
|
||||
function convertWithReadability(
|
||||
rawHtml: string,
|
||||
cleanedHtml: string,
|
||||
url: string,
|
||||
baseMetadata: HtmlConversionMetadata,
|
||||
): HtmlToMarkdownResult {
|
||||
const dom = new JSDOM(cleanedHtml, { url });
|
||||
const document = dom.window.document;
|
||||
const article = new Readability(document).parse();
|
||||
|
||||
const contentHtml =
|
||||
article?.content?.trim() ??
|
||||
document.querySelector("main")?.innerHTML?.trim() ??
|
||||
document.body?.innerHTML?.trim() ??
|
||||
"";
|
||||
|
||||
let markdown = contentHtml ? convertHtmlFragmentToMarkdown(contentHtml) : "";
|
||||
if (!markdown) {
|
||||
markdown = fallbackPlainText(cleanedHtml);
|
||||
}
|
||||
|
||||
const metadata: HtmlConversionMetadata = {
|
||||
...baseMetadata,
|
||||
title: pickString(article?.title, baseMetadata.title),
|
||||
summary: pickString(article?.excerpt, baseMetadata.summary),
|
||||
author: pickString(article?.byline, baseMetadata.author),
|
||||
};
|
||||
|
||||
const media = collectMediaFromMarkdown(markdown);
|
||||
if (metadata.coverImage) {
|
||||
media.unshift({
|
||||
url: metadata.coverImage,
|
||||
kind: "image",
|
||||
role: "cover",
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
metadata,
|
||||
markdown: trimKnownBoilerplate(markdown),
|
||||
rawHtml,
|
||||
cleanedHtml,
|
||||
media,
|
||||
conversionMethod: article?.content ? "legacy:readability" : "legacy:body",
|
||||
};
|
||||
}
|
||||
|
||||
export async function convertHtmlToMarkdown(
|
||||
html: string,
|
||||
url: string,
|
||||
options: ConvertHtmlToMarkdownOptions = {},
|
||||
): Promise<HtmlToMarkdownResult> {
|
||||
const capturedAt = new Date().toISOString();
|
||||
const baseMetadata = extractPageMetadata(html, url, capturedAt);
|
||||
|
||||
let cleanedHtml = html;
|
||||
try {
|
||||
cleanedHtml = cleanHtml(html, url, {
|
||||
removeBase64Images: !options.preserveBase64Images,
|
||||
});
|
||||
} catch {
|
||||
cleanedHtml = html;
|
||||
}
|
||||
|
||||
let selectedResult: HtmlToMarkdownResult;
|
||||
const defuddleResult = await tryDefuddleConversion(cleanedHtml, url, baseMetadata);
|
||||
if (defuddleResult.ok) {
|
||||
if (shouldCompareWithFallback(defuddleResult.result.markdown)) {
|
||||
const fallbackResult = convertWithReadability(html, cleanedHtml, url, baseMetadata);
|
||||
if (
|
||||
scoreMarkdownQuality(fallbackResult.markdown) >
|
||||
scoreMarkdownQuality(defuddleResult.result.markdown) + LOCAL_FALLBACK_SCORE_DELTA
|
||||
) {
|
||||
selectedResult = {
|
||||
...fallbackResult,
|
||||
fallbackReason: "Readability/Turndown produced higher-quality markdown than Defuddle",
|
||||
};
|
||||
} else {
|
||||
selectedResult = {
|
||||
...defuddleResult.result,
|
||||
rawHtml: html,
|
||||
cleanedHtml,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
selectedResult = {
|
||||
...defuddleResult.result,
|
||||
rawHtml: html,
|
||||
cleanedHtml,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
selectedResult = {
|
||||
...convertWithReadability(html, cleanedHtml, url, baseMetadata),
|
||||
fallbackReason: defuddleResult.reason,
|
||||
};
|
||||
}
|
||||
|
||||
if (!shouldTryRemoteMarkdownFallback(selectedResult.markdown, cleanedHtml, options)) {
|
||||
return selectedResult;
|
||||
}
|
||||
|
||||
const remoteDefuddleResult = await tryDefuddleApiConversion(cleanedHtml, url, baseMetadata);
|
||||
if (!remoteDefuddleResult.ok || !shouldPreferRemoteMarkdown(selectedResult, remoteDefuddleResult.result, cleanedHtml)) {
|
||||
return selectedResult;
|
||||
}
|
||||
|
||||
return {
|
||||
...remoteDefuddleResult.result,
|
||||
rawHtml: html,
|
||||
cleanedHtml,
|
||||
fallbackReason: buildRemoteFallbackReason(selectedResult, cleanedHtml),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
import { normalizeMarkdownMediaLinks } from "../media/markdown-media";
|
||||
import type { ContentBlock, ExtractedDocument } from "./document";
|
||||
|
||||
const turndownService = new TurndownService({
|
||||
codeBlockStyle: "fenced",
|
||||
headingStyle: "atx",
|
||||
bulletListMarker: "-",
|
||||
});
|
||||
|
||||
turndownService.use(gfm);
|
||||
|
||||
function renderBlock(block: ContentBlock): string {
|
||||
switch (block.type) {
|
||||
case "paragraph":
|
||||
return block.text.trim();
|
||||
case "heading":
|
||||
return `${"#".repeat(Math.min(Math.max(block.depth, 1), 6))} ${block.text.trim()}`;
|
||||
case "list":
|
||||
return block.items
|
||||
.map((item, index) => (block.ordered ? `${index + 1}. ${item.trim()}` : `- ${item.trim()}`))
|
||||
.join("\n");
|
||||
case "quote":
|
||||
return block.text
|
||||
.split("\n")
|
||||
.map((line) => `> ${line}`)
|
||||
.join("\n");
|
||||
case "code":
|
||||
return `\`\`\`${block.language ?? ""}\n${block.code.trimEnd()}\n\`\`\``;
|
||||
case "image":
|
||||
return ``;
|
||||
case "html":
|
||||
return turndownService.turndown(block.html).trim();
|
||||
case "markdown":
|
||||
return block.markdown.trim();
|
||||
}
|
||||
}
|
||||
|
||||
function isDefinedValue(value: unknown): boolean {
|
||||
return value !== undefined && value !== null && value !== "";
|
||||
}
|
||||
|
||||
function renderFrontmatterValue(value: unknown): string {
|
||||
if (typeof value === "string") {
|
||||
if (value.includes("\n")) {
|
||||
return `|-\n${value
|
||||
.replace(/\r\n/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n")}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
if (typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function renderFrontmatter(document: ExtractedDocument): string {
|
||||
const fields = new Map<string, unknown>();
|
||||
const preferredOrder = [
|
||||
"title",
|
||||
"url",
|
||||
"requestedUrl",
|
||||
"author",
|
||||
"authorName",
|
||||
"authorUsername",
|
||||
"authorUrl",
|
||||
"coverImage",
|
||||
"siteName",
|
||||
"publishedAt",
|
||||
"summary",
|
||||
"adapter",
|
||||
];
|
||||
|
||||
fields.set("title", document.title);
|
||||
fields.set("url", document.canonicalUrl ?? document.url);
|
||||
fields.set("requestedUrl", document.requestedUrl ?? document.url);
|
||||
fields.set("author", document.author);
|
||||
fields.set("siteName", document.siteName);
|
||||
fields.set("publishedAt", document.publishedAt);
|
||||
fields.set("summary", document.summary);
|
||||
fields.set("adapter", document.adapter);
|
||||
|
||||
for (const [key, value] of Object.entries(document.metadata ?? {})) {
|
||||
if (!fields.has(key)) {
|
||||
fields.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
const orderedKeys = [
|
||||
...preferredOrder.filter((key) => fields.has(key)),
|
||||
...Array.from(fields.keys()).filter((key) => !preferredOrder.includes(key)).sort(),
|
||||
];
|
||||
|
||||
const lines = orderedKeys
|
||||
.map((key) => [key, fields.get(key)] as const)
|
||||
.filter(([, value]) => isDefinedValue(value))
|
||||
.map(([key, value]) => `${key}: ${renderFrontmatterValue(value)}`);
|
||||
|
||||
if (lines.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return `---\n${lines.join("\n")}\n---`;
|
||||
}
|
||||
|
||||
function cleanMarkdown(markdown: string): string {
|
||||
return normalizeMarkdownMediaLinks(markdown.replace(/\n{3,}/g, "\n\n").trim());
|
||||
}
|
||||
|
||||
function normalizeComparableTitle(value: string): string {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/^>\s*/, "")
|
||||
.replace(/^#+\s+/, "")
|
||||
.replace(/(?:\.{3}|…)\s*$/, "");
|
||||
}
|
||||
|
||||
function bodyStartsWithTitle(body: string, title: string): boolean {
|
||||
const firstMeaningfulLine = body
|
||||
.replace(/\r\n/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line));
|
||||
|
||||
if (!firstMeaningfulLine) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const comparableTitle = normalizeComparableTitle(title);
|
||||
const comparableFirstLine = normalizeComparableTitle(firstMeaningfulLine);
|
||||
if (!comparableTitle || !comparableFirstLine) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
comparableFirstLine === comparableTitle ||
|
||||
comparableFirstLine.startsWith(comparableTitle) ||
|
||||
comparableTitle.startsWith(comparableFirstLine)
|
||||
);
|
||||
}
|
||||
|
||||
export function renderMarkdown(document: ExtractedDocument): string {
|
||||
const sections: string[] = [];
|
||||
const frontmatter = renderFrontmatter(document);
|
||||
|
||||
if (frontmatter) {
|
||||
sections.push(frontmatter);
|
||||
}
|
||||
|
||||
const body = document.content
|
||||
.map((block) => renderBlock(block))
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
if (document.title && !bodyStartsWithTitle(body, document.title)) {
|
||||
sections.push(`# ${document.title}`);
|
||||
}
|
||||
|
||||
if (body) {
|
||||
sections.push(body);
|
||||
}
|
||||
|
||||
return cleanMarkdown(sections.join("\n\n"));
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import path from "node:path";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
buildFileName,
|
||||
isDataUri,
|
||||
normalizeContentType,
|
||||
normalizeMediaUrl,
|
||||
resolveExtensionFromContentType,
|
||||
resolveExtensionFromUrl,
|
||||
resolveMediaKind,
|
||||
resolveOutputExtension,
|
||||
toPosixPath,
|
||||
} from "./media-utils";
|
||||
import type { MediaAsset, MediaDownloadRequest, MediaDownloadResult, MediaKind } from "./types";
|
||||
|
||||
const DOWNLOAD_USER_AGENT =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
|
||||
|
||||
function parseBase64DataUri(rawUrl: string): { contentType: string; bytes: Buffer } | null {
|
||||
const match = rawUrl.match(/^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/i);
|
||||
if (!match?.[1] || !match[2]) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const contentType = normalizeContentType(match[1]);
|
||||
if (!contentType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const bytes = Buffer.from(match[2].replace(/\s+/g, ""), "base64");
|
||||
if (bytes.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return { contentType, bytes };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function dedupeMedia(media: MediaAsset[]): MediaAsset[] {
|
||||
const deduped: MediaAsset[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of media) {
|
||||
const normalizedUrl = normalizeMediaUrl(item.url);
|
||||
if (!normalizedUrl || seen.has(normalizedUrl)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(normalizedUrl);
|
||||
deduped.push({
|
||||
...item,
|
||||
url: normalizedUrl,
|
||||
});
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function toRelativePath(fromDir: string, absoluteTarget: string): string {
|
||||
const relative = path.relative(fromDir, absoluteTarget) || path.basename(absoluteTarget);
|
||||
return toPosixPath(relative);
|
||||
}
|
||||
|
||||
export async function downloadMediaAssets(
|
||||
request: MediaDownloadRequest,
|
||||
): Promise<MediaDownloadResult> {
|
||||
const dedupedMedia = dedupeMedia(request.media);
|
||||
const absoluteOutputPath = path.resolve(request.outputPath);
|
||||
const markdownDir = path.dirname(absoluteOutputPath);
|
||||
const baseDir = request.mediaDir ? path.resolve(request.mediaDir) : markdownDir;
|
||||
const replacements: MediaDownloadResult["replacements"] = [];
|
||||
|
||||
let downloadedImages = 0;
|
||||
let downloadedVideos = 0;
|
||||
|
||||
for (const asset of dedupedMedia) {
|
||||
try {
|
||||
let sourceUrl = normalizeMediaUrl(asset.url);
|
||||
let contentType = "";
|
||||
let extension: string | undefined;
|
||||
let kind: MediaKind | undefined;
|
||||
let bytes: Buffer | null = null;
|
||||
|
||||
if (isDataUri(asset.url)) {
|
||||
const parsed = parseBase64DataUri(asset.url);
|
||||
if (!parsed) {
|
||||
request.log.warn(`Skipping unsupported embedded media: ${asset.url.slice(0, 32)}...`);
|
||||
continue;
|
||||
}
|
||||
|
||||
contentType = parsed.contentType;
|
||||
extension =
|
||||
resolveExtensionFromContentType(contentType) ??
|
||||
resolveExtensionFromUrl(asset.fileNameHint ?? "");
|
||||
kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
|
||||
bytes = parsed.bytes;
|
||||
} else {
|
||||
const response = await fetch(sourceUrl, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers: {
|
||||
"user-agent": DOWNLOAD_USER_AGENT,
|
||||
...(asset.headers ?? {}),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
request.log.warn(`Skipping media (${response.status}): ${asset.url}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
sourceUrl = normalizeMediaUrl(response.url || sourceUrl);
|
||||
contentType = normalizeContentType(response.headers.get("content-type"));
|
||||
extension =
|
||||
resolveExtensionFromUrl(sourceUrl) ??
|
||||
resolveExtensionFromUrl(asset.url) ??
|
||||
resolveExtensionFromUrl(asset.fileNameHint ?? "");
|
||||
kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
|
||||
bytes = Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
if (!kind || !bytes) {
|
||||
request.log.debug(`Skipping media with unresolved kind: ${asset.url}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const outputExtension = resolveOutputExtension(contentType, extension, kind);
|
||||
const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1;
|
||||
const dirName = kind === "image" ? "imgs" : "videos";
|
||||
const targetDir = path.join(baseDir, dirName);
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
|
||||
const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension, asset.fileNameHint);
|
||||
const absolutePath = path.join(targetDir, fileName);
|
||||
await writeFile(absolutePath, bytes);
|
||||
|
||||
replacements.push({
|
||||
url: asset.url,
|
||||
localPath: toRelativePath(markdownDir, absolutePath),
|
||||
absolutePath,
|
||||
kind,
|
||||
});
|
||||
|
||||
if (kind === "image") {
|
||||
downloadedImages = nextIndex;
|
||||
} else {
|
||||
downloadedVideos = nextIndex;
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
request.log.warn(`Failed to download media ${asset.url}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
replacements,
|
||||
downloadedImages,
|
||||
downloadedVideos,
|
||||
imageDir: downloadedImages > 0 ? path.join(baseDir, "imgs") : null,
|
||||
videoDir: downloadedVideos > 0 ? path.join(baseDir, "videos") : null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,458 @@
|
||||
import remarkGfm from "remark-gfm";
|
||||
import remarkParse from "remark-parse";
|
||||
import { unified } from "unified";
|
||||
import type { ContentBlock, ExtractedDocument } from "../extract/document";
|
||||
import {
|
||||
isDataUri,
|
||||
normalizeContentType,
|
||||
normalizeMediaUrl,
|
||||
resolveExtensionFromContentType,
|
||||
resolveExtensionFromUrl,
|
||||
resolveKindFromExtension,
|
||||
} from "./media-utils";
|
||||
import type { MediaAsset, MediaReplacement } from "./types";
|
||||
|
||||
const MARKDOWN_LINK_RE =
|
||||
/(!?\[[^\]\n]*\])\((<)?((?:https?:\/\/[^)\s>]+)|(?:data:[^)>\s]+))(>)?\)/g;
|
||||
const FRONTMATTER_COVER_RE = /^(coverImage:\s*")((?:https?:\/\/[^"]+)|(?:data:[^"]+))(")/m;
|
||||
const RAW_URL_RE = /(?:https?:\/\/[^\s<>"')\]]+|data:[^\s<>"')\]]+)/g;
|
||||
|
||||
interface MarkdownAstNode {
|
||||
type: string;
|
||||
url?: string | null;
|
||||
alt?: string | null;
|
||||
title?: string | null;
|
||||
value?: string | null;
|
||||
children?: MarkdownAstNode[];
|
||||
position?: {
|
||||
start?: { offset?: number | null };
|
||||
end?: { offset?: number | null };
|
||||
};
|
||||
}
|
||||
|
||||
interface MarkdownReplacementRange {
|
||||
start: number;
|
||||
end: number;
|
||||
value: string;
|
||||
}
|
||||
|
||||
function inferMediaKindFromLabel(label: string, rawUrl: string): "image" | "video" | undefined {
|
||||
if (label.startsWith("![")) {
|
||||
return "image";
|
||||
}
|
||||
|
||||
const normalizedLabel = label.replace(/[!\[\]]/g, "").trim().toLowerCase();
|
||||
if (/\b(video|animated[_ -]?gif|gif)\b/.test(normalizedLabel)) {
|
||||
return "video";
|
||||
}
|
||||
|
||||
if (isDataUri(rawUrl)) {
|
||||
const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";")));
|
||||
return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined;
|
||||
}
|
||||
|
||||
return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl));
|
||||
}
|
||||
|
||||
function inferMediaKindFromRawUrl(rawUrl: string): "image" | "video" | undefined {
|
||||
if (isDataUri(rawUrl)) {
|
||||
const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";")));
|
||||
return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined;
|
||||
}
|
||||
|
||||
return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl));
|
||||
}
|
||||
|
||||
function pushMedia(assets: MediaAsset[], seen: Set<string>, media: MediaAsset): void {
|
||||
const normalizedUrl = normalizeMediaUrl(media.url);
|
||||
if (!normalizedUrl || seen.has(normalizedUrl)) {
|
||||
return;
|
||||
}
|
||||
seen.add(normalizedUrl);
|
||||
assets.push({
|
||||
...media,
|
||||
url: normalizedUrl,
|
||||
});
|
||||
}
|
||||
|
||||
function getNodeOffsets(node: MarkdownAstNode): { start: number; end: number } | null {
|
||||
const start = node.position?.start?.offset;
|
||||
const end = node.position?.end?.offset;
|
||||
if (typeof start !== "number" || typeof end !== "number" || start < 0 || end < start) {
|
||||
return null;
|
||||
}
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
function escapeMarkdownLabel(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/\[/g, "\\[").replace(/\]/g, "\\]");
|
||||
}
|
||||
|
||||
function escapeMarkdownTitle(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
||||
}
|
||||
|
||||
function formatMarkdownDestination(url: string): string {
|
||||
return /[\s()<>]/.test(url) ? `<${url}>` : url;
|
||||
}
|
||||
|
||||
function serializeImageNode(node: MarkdownAstNode): string {
|
||||
const rawUrl = node.url ?? "";
|
||||
const normalizedUrl = normalizeMediaUrl(rawUrl);
|
||||
const alt = escapeMarkdownLabel(node.alt ?? "");
|
||||
const title = node.title ? ` "${escapeMarkdownTitle(node.title)}"` : "";
|
||||
return `}${title})`;
|
||||
}
|
||||
|
||||
function serializeLinkedImageNode(linkNode: MarkdownAstNode, imageNode: MarkdownAstNode): string {
|
||||
const imageMarkdown = serializeImageNode(imageNode);
|
||||
const imageUrl = normalizeMediaUrl(imageNode.url ?? "");
|
||||
const linkUrl = normalizeMediaUrl(linkNode.url ?? "");
|
||||
|
||||
if (!linkUrl || linkUrl === imageUrl) {
|
||||
return imageMarkdown;
|
||||
}
|
||||
|
||||
const title = linkNode.title ? ` "${escapeMarkdownTitle(linkNode.title)}"` : "";
|
||||
return `[${imageMarkdown}](${formatMarkdownDestination(linkUrl)}${title})`;
|
||||
}
|
||||
|
||||
function isParagraphWithSingleText(node: MarkdownAstNode | undefined, expectedValue: string): boolean {
|
||||
if (node?.type !== "paragraph" || node.children?.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const child = node.children[0];
|
||||
return child?.type === "text" && child.value?.trim() === expectedValue;
|
||||
}
|
||||
|
||||
function getSingleImageFromParagraph(node: MarkdownAstNode | undefined): MarkdownAstNode | null {
|
||||
if (node?.type !== "paragraph" || node.children?.length !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return node.children[0]?.type === "image" ? node.children[0] : null;
|
||||
}
|
||||
|
||||
function extractBrokenLinkedImageDestination(node: MarkdownAstNode | undefined): string | null {
|
||||
if (node?.type !== "paragraph") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const children = node.children ?? [];
|
||||
if (children.length !== 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [prefix, linkNode, suffix] = children;
|
||||
if (prefix?.type !== "text" || prefix.value?.trim() !== "](") {
|
||||
return null;
|
||||
}
|
||||
if (linkNode?.type !== "link" || !linkNode.url) {
|
||||
return null;
|
||||
}
|
||||
if (suffix?.type !== "text" || suffix.value?.trim() !== ")") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return linkNode.url;
|
||||
}
|
||||
|
||||
function collectLinkedImageReplacements(
|
||||
node: MarkdownAstNode,
|
||||
replacements: MarkdownReplacementRange[],
|
||||
): void {
|
||||
const children = node.children ?? [];
|
||||
|
||||
if (node.type === "link" && children.length === 1 && children[0]?.type === "image") {
|
||||
const offsets = getNodeOffsets(node);
|
||||
if (offsets) {
|
||||
replacements.push({
|
||||
start: offsets.start,
|
||||
end: offsets.end,
|
||||
value: serializeLinkedImageNode(node, children[0]),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (const child of children) {
|
||||
collectLinkedImageReplacements(child, replacements);
|
||||
}
|
||||
}
|
||||
|
||||
function collectBrokenLinkedImageReplacements(
|
||||
node: MarkdownAstNode,
|
||||
replacements: MarkdownReplacementRange[],
|
||||
): void {
|
||||
const children = node.children ?? [];
|
||||
for (let index = 0; index <= children.length - 3; index += 1) {
|
||||
const openParagraph = children[index];
|
||||
const imageParagraph = children[index + 1];
|
||||
const closeParagraph = children[index + 2];
|
||||
|
||||
if (!isParagraphWithSingleText(openParagraph, "[")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const imageNode = getSingleImageFromParagraph(imageParagraph);
|
||||
if (!imageNode) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const linkUrl = extractBrokenLinkedImageDestination(closeParagraph);
|
||||
if (!linkUrl) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const start = openParagraph.position?.start?.offset;
|
||||
const end = closeParagraph.position?.end?.offset;
|
||||
if (typeof start !== "number" || typeof end !== "number" || end < start) {
|
||||
continue;
|
||||
}
|
||||
|
||||
replacements.push({
|
||||
start,
|
||||
end,
|
||||
value: serializeLinkedImageNode({ type: "link", url: linkUrl }, imageNode),
|
||||
});
|
||||
|
||||
index += 2;
|
||||
}
|
||||
|
||||
for (const child of children) {
|
||||
collectBrokenLinkedImageReplacements(child, replacements);
|
||||
}
|
||||
}
|
||||
|
||||
function applyReplacements(source: string, replacements: MarkdownReplacementRange[]): string {
|
||||
if (replacements.length === 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
let result = source;
|
||||
const sorted = [...replacements].sort((left, right) => right.start - left.start);
|
||||
for (const replacement of sorted) {
|
||||
result = `${result.slice(0, replacement.start)}${replacement.value}${result.slice(replacement.end)}`;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeLinkedImageMarkdown(markdown: string): string {
|
||||
let tree: MarkdownAstNode;
|
||||
try {
|
||||
tree = unified().use(remarkParse).use(remarkGfm).parse(markdown) as MarkdownAstNode;
|
||||
} catch {
|
||||
return markdown;
|
||||
}
|
||||
|
||||
const replacements: MarkdownReplacementRange[] = [];
|
||||
collectLinkedImageReplacements(tree, replacements);
|
||||
collectBrokenLinkedImageReplacements(tree, replacements);
|
||||
return applyReplacements(markdown, replacements);
|
||||
}
|
||||
|
||||
export function normalizeMarkdownMediaLinks(markdown: string): string {
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, openAngle, rawUrl, closeAngle) => {
|
||||
const normalizedUrl = normalizeMediaUrl(rawUrl);
|
||||
if (normalizedUrl === rawUrl) {
|
||||
return full;
|
||||
}
|
||||
return `${label}(${openAngle ?? ""}${normalizedUrl}${closeAngle ?? ""})`;
|
||||
});
|
||||
|
||||
result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
|
||||
const normalizedUrl = normalizeMediaUrl(rawUrl);
|
||||
if (normalizedUrl === rawUrl) {
|
||||
return full;
|
||||
}
|
||||
return `${prefix}${normalizedUrl}${suffix}`;
|
||||
});
|
||||
|
||||
RAW_URL_RE.lastIndex = 0;
|
||||
result = result.replace(RAW_URL_RE, (rawUrl) => normalizeMediaUrl(rawUrl));
|
||||
return normalizeLinkedImageMarkdown(result);
|
||||
}
|
||||
|
||||
export function collectMediaFromText(
|
||||
text: string,
|
||||
options: {
|
||||
role?: MediaAsset["role"];
|
||||
defaultKind?: MediaAsset["kind"];
|
||||
seen?: Set<string>;
|
||||
into?: MediaAsset[];
|
||||
} = {},
|
||||
): MediaAsset[] {
|
||||
const assets = options.into ?? [];
|
||||
const seen = options.seen ?? new Set<string>();
|
||||
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let linkMatch: RegExpExecArray | null;
|
||||
while ((linkMatch = MARKDOWN_LINK_RE.exec(text))) {
|
||||
const label = linkMatch[1] ?? "";
|
||||
const rawUrl = linkMatch[3] ?? "";
|
||||
const kind = inferMediaKindFromLabel(label, rawUrl) ?? options.defaultKind;
|
||||
if (!kind) {
|
||||
continue;
|
||||
}
|
||||
pushMedia(assets, seen, {
|
||||
url: rawUrl,
|
||||
kind,
|
||||
role: options.role ?? "inline",
|
||||
});
|
||||
}
|
||||
|
||||
RAW_URL_RE.lastIndex = 0;
|
||||
let rawMatch: RegExpExecArray | null;
|
||||
while ((rawMatch = RAW_URL_RE.exec(text))) {
|
||||
const rawUrl = rawMatch[0] ?? "";
|
||||
const kind = inferMediaKindFromRawUrl(rawUrl) ?? options.defaultKind;
|
||||
if (!kind) {
|
||||
continue;
|
||||
}
|
||||
pushMedia(assets, seen, {
|
||||
url: rawUrl,
|
||||
kind,
|
||||
role: options.role ?? "inline",
|
||||
});
|
||||
}
|
||||
|
||||
return assets;
|
||||
}
|
||||
|
||||
function collectMediaFromBlock(
|
||||
block: ContentBlock,
|
||||
assets: MediaAsset[],
|
||||
seen: Set<string>,
|
||||
): void {
|
||||
switch (block.type) {
|
||||
case "image":
|
||||
pushMedia(assets, seen, {
|
||||
url: block.url,
|
||||
kind: "image",
|
||||
role: "inline",
|
||||
alt: block.alt,
|
||||
});
|
||||
return;
|
||||
case "html":
|
||||
case "markdown":
|
||||
collectMediaFromText(block.type === "html" ? block.html : block.markdown, {
|
||||
role: "inline",
|
||||
seen,
|
||||
into: assets,
|
||||
});
|
||||
return;
|
||||
case "paragraph":
|
||||
case "quote":
|
||||
collectMediaFromText(block.text, {
|
||||
role: "inline",
|
||||
seen,
|
||||
into: assets,
|
||||
});
|
||||
return;
|
||||
case "list":
|
||||
for (const item of block.items) {
|
||||
collectMediaFromText(item, {
|
||||
role: "attachment",
|
||||
seen,
|
||||
into: assets,
|
||||
});
|
||||
}
|
||||
return;
|
||||
case "heading":
|
||||
case "code":
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export function collectMediaFromDocument(document: ExtractedDocument): MediaAsset[] {
|
||||
const assets: MediaAsset[] = [];
|
||||
const seen = new Set<string>();
|
||||
const coverImage =
|
||||
typeof document.metadata?.coverImage === "string" ? document.metadata.coverImage : undefined;
|
||||
|
||||
if (coverImage) {
|
||||
pushMedia(assets, seen, {
|
||||
url: coverImage,
|
||||
kind: "image",
|
||||
role: "cover",
|
||||
});
|
||||
}
|
||||
|
||||
for (const block of document.content) {
|
||||
collectMediaFromBlock(block, assets, seen);
|
||||
}
|
||||
|
||||
return assets;
|
||||
}
|
||||
|
||||
export function collectMediaFromMarkdown(markdown: string): MediaAsset[] {
|
||||
const assets: MediaAsset[] = [];
|
||||
const seen = new Set<string>();
|
||||
const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (fmMatch) {
|
||||
const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
|
||||
if (coverMatch?.[2]) {
|
||||
pushMedia(assets, seen, {
|
||||
url: coverMatch[2],
|
||||
kind: "image",
|
||||
role: "cover",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
collectMediaFromText(markdown, { seen, into: assets });
|
||||
return assets;
|
||||
}
|
||||
|
||||
export function rewriteMarkdownMediaLinks(
|
||||
markdown: string,
|
||||
replacements: MediaReplacement[],
|
||||
): string {
|
||||
if (replacements.length === 0) {
|
||||
return markdown;
|
||||
}
|
||||
|
||||
const replacementMap = new Map<string, string>();
|
||||
for (const item of replacements) {
|
||||
replacementMap.set(item.url, item.localPath);
|
||||
replacementMap.set(normalizeMediaUrl(item.url), item.localPath);
|
||||
}
|
||||
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, _openAngle, rawUrl) => {
|
||||
const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl));
|
||||
if (!replacement) {
|
||||
return full;
|
||||
}
|
||||
return `${label}(${replacement})`;
|
||||
});
|
||||
|
||||
result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
|
||||
const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl));
|
||||
if (!replacement) {
|
||||
return full;
|
||||
}
|
||||
return `${prefix}${replacement}${suffix}`;
|
||||
});
|
||||
|
||||
for (const { url, localPath } of replacements) {
|
||||
result = result.split(url).join(localPath);
|
||||
const normalizedUrl = normalizeMediaUrl(url);
|
||||
if (normalizedUrl !== url) {
|
||||
result = result.split(normalizedUrl).join(localPath);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export function resolveDataUriExtension(rawUrl: string): string | undefined {
|
||||
if (!isDataUri(rawUrl)) {
|
||||
return undefined;
|
||||
}
|
||||
const separatorIndex = rawUrl.indexOf(";");
|
||||
const contentType = normalizeContentType(rawUrl.slice(5, separatorIndex === -1 ? undefined : separatorIndex));
|
||||
return resolveExtensionFromContentType(contentType);
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
import path from "node:path";
|
||||
import type { MediaKind } from "./types";
|
||||
|
||||
const IMAGE_EXTENSIONS = new Set([
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"png",
|
||||
"webp",
|
||||
"gif",
|
||||
"bmp",
|
||||
"avif",
|
||||
"heic",
|
||||
"heif",
|
||||
"svg",
|
||||
]);
|
||||
|
||||
const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]);
|
||||
|
||||
const MIME_EXTENSION_MAP: Record<string, string> = {
|
||||
"image/jpeg": "jpg",
|
||||
"image/jpg": "jpg",
|
||||
"image/png": "png",
|
||||
"image/webp": "webp",
|
||||
"image/gif": "gif",
|
||||
"image/bmp": "bmp",
|
||||
"image/avif": "avif",
|
||||
"image/heic": "heic",
|
||||
"image/heif": "heif",
|
||||
"image/svg+xml": "svg",
|
||||
"video/mp4": "mp4",
|
||||
"video/webm": "webm",
|
||||
"video/quicktime": "mov",
|
||||
"video/x-m4v": "m4v",
|
||||
};
|
||||
|
||||
export function normalizeContentType(raw: string | null): string {
|
||||
return raw?.split(";")[0]?.trim().toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
export function normalizeExtension(raw: string | undefined | null): string | undefined {
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = raw.replace(/^\./, "").trim().toLowerCase();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
if (trimmed === "jpeg" || trimmed === "jpg") {
|
||||
return "jpg";
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function resolveExtensionFromUrl(rawUrl: string): string | undefined {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname));
|
||||
if (extFromPath) {
|
||||
return extFromPath;
|
||||
}
|
||||
const extFromFormat = normalizeExtension(parsed.searchParams.get("format"));
|
||||
if (extFromFormat) {
|
||||
return extFromFormat;
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveExtensionFromContentType(contentType: string): string | undefined {
|
||||
return normalizeExtension(MIME_EXTENSION_MAP[contentType]);
|
||||
}
|
||||
|
||||
export function resolveKindFromContentType(contentType: string): MediaKind | undefined {
|
||||
if (!contentType) {
|
||||
return undefined;
|
||||
}
|
||||
if (contentType.startsWith("image/")) {
|
||||
return "image";
|
||||
}
|
||||
if (contentType.startsWith("video/")) {
|
||||
return "video";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveKindFromExtension(extension: string | undefined): MediaKind | undefined {
|
||||
if (!extension) {
|
||||
return undefined;
|
||||
}
|
||||
if (IMAGE_EXTENSIONS.has(extension)) {
|
||||
return "image";
|
||||
}
|
||||
if (VIDEO_EXTENSIONS.has(extension)) {
|
||||
return "video";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveMediaKind(
|
||||
rawUrl: string,
|
||||
contentType: string,
|
||||
extension: string | undefined,
|
||||
hint?: MediaKind,
|
||||
): MediaKind | undefined {
|
||||
const kindFromType = resolveKindFromContentType(contentType);
|
||||
if (kindFromType) {
|
||||
return kindFromType;
|
||||
}
|
||||
|
||||
const kindFromExtension = resolveKindFromExtension(extension);
|
||||
if (kindFromExtension) {
|
||||
return kindFromExtension;
|
||||
}
|
||||
|
||||
if (contentType && contentType !== "application/octet-stream") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (hint) {
|
||||
return hint;
|
||||
}
|
||||
|
||||
if (rawUrl.startsWith("data:image/")) {
|
||||
return "image";
|
||||
}
|
||||
|
||||
if (rawUrl.startsWith("data:video/")) {
|
||||
return "video";
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveOutputExtension(
|
||||
contentType: string,
|
||||
extension: string | undefined,
|
||||
kind: MediaKind,
|
||||
): string {
|
||||
const fromMime = resolveExtensionFromContentType(contentType);
|
||||
if (fromMime) {
|
||||
return fromMime;
|
||||
}
|
||||
const normalized = normalizeExtension(extension);
|
||||
if (normalized) {
|
||||
return normalized;
|
||||
}
|
||||
return kind === "video" ? "mp4" : "jpg";
|
||||
}
|
||||
|
||||
export function isDataUri(value: string): boolean {
|
||||
return value.startsWith("data:");
|
||||
}
|
||||
|
||||
export function safeDecodeURIComponent(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function extractEmbeddedUrl(value: string): string | undefined {
|
||||
const encodedMatch = value.match(/https?%3A%2F%2F.+$/i)?.[0];
|
||||
if (encodedMatch) {
|
||||
const decoded = safeDecodeURIComponent(encodedMatch);
|
||||
try {
|
||||
return new URL(decoded).href;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const literalMatch = value.match(/https?:\/\/.+$/i)?.[0];
|
||||
if (!literalMatch) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
return new URL(literalMatch).href;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeMediaUrl(rawUrl: string): string {
|
||||
if (isDataUri(rawUrl)) {
|
||||
return rawUrl;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
const hostname = parsed.hostname.toLowerCase();
|
||||
|
||||
if (hostname === "substackcdn.com" || hostname.endsWith(".substackcdn.com")) {
|
||||
const embeddedUrl = extractEmbeddedUrl(`${parsed.pathname}${parsed.search}`);
|
||||
if (embeddedUrl) {
|
||||
return embeddedUrl;
|
||||
}
|
||||
}
|
||||
|
||||
return parsed.href;
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export function sanitizeFileSegment(input: string): string {
|
||||
return input
|
||||
.replace(/[^a-zA-Z0-9_-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^[-_]+|[-_]+$/g, "")
|
||||
.slice(0, 48);
|
||||
}
|
||||
|
||||
export function resolveFileStem(rawUrl: string, extension: string, fileNameHint?: string): string {
|
||||
const hintBase = fileNameHint?.trim();
|
||||
if (hintBase) {
|
||||
const parsed = path.posix.parse(hintBase);
|
||||
const stem = parsed.name || parsed.base;
|
||||
return sanitizeFileSegment(stem);
|
||||
}
|
||||
|
||||
if (isDataUri(rawUrl)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
const base = path.posix.basename(parsed.pathname);
|
||||
if (!base) {
|
||||
return "";
|
||||
}
|
||||
const decodedBase = safeDecodeURIComponent(base);
|
||||
const normalizedExtension = normalizeExtension(extension);
|
||||
const stripExtension = normalizedExtension ? new RegExp(`\\.${normalizedExtension}$`, "i") : null;
|
||||
const rawStem = stripExtension ? decodedBase.replace(stripExtension, "") : decodedBase;
|
||||
return sanitizeFileSegment(rawStem);
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFileName(
|
||||
kind: MediaKind,
|
||||
index: number,
|
||||
sourceUrl: string,
|
||||
extension: string,
|
||||
fileNameHint?: string,
|
||||
): string {
|
||||
const stem = resolveFileStem(sourceUrl, extension, fileNameHint);
|
||||
const prefix = kind === "image" ? "img" : "video";
|
||||
const serial = String(index).padStart(3, "0");
|
||||
const suffix = stem ? `-${stem}` : "";
|
||||
return `${prefix}-${serial}${suffix}.${extension}`;
|
||||
}
|
||||
|
||||
export function toPosixPath(value: string): string {
|
||||
return value.split(path.sep).join(path.posix.sep);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import type { Logger } from "../utils/logger";
|
||||
|
||||
export type MediaKind = "image" | "video";
|
||||
|
||||
export interface MediaAsset {
|
||||
url: string;
|
||||
kind?: MediaKind;
|
||||
role?: "cover" | "inline" | "attachment";
|
||||
alt?: string;
|
||||
fileNameHint?: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface MediaReplacement {
|
||||
url: string;
|
||||
localPath: string;
|
||||
absolutePath: string;
|
||||
kind: MediaKind;
|
||||
}
|
||||
|
||||
export interface MediaDownloadRequest {
|
||||
media: MediaAsset[];
|
||||
outputPath: string;
|
||||
mediaDir?: string;
|
||||
log: Logger;
|
||||
}
|
||||
|
||||
export interface MediaDownloadResult {
|
||||
replacements: MediaReplacement[];
|
||||
downloadedImages: number;
|
||||
downloadedVideos: number;
|
||||
imageDir: string | null;
|
||||
videoDir: string | null;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
declare module "defuddle/node" {
|
||||
export interface DefuddleResponse {
|
||||
content?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
author?: string;
|
||||
published?: string;
|
||||
image?: string;
|
||||
language?: string;
|
||||
}
|
||||
|
||||
export interface DefuddleOptions {
|
||||
markdown?: boolean;
|
||||
}
|
||||
|
||||
export function Defuddle(
|
||||
input:
|
||||
| Document
|
||||
| string
|
||||
| {
|
||||
window: {
|
||||
document: Document;
|
||||
location: {
|
||||
href: string;
|
||||
};
|
||||
};
|
||||
},
|
||||
url?: string,
|
||||
options?: DefuddleOptions,
|
||||
): Promise<DefuddleResponse>;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
declare module "turndown" {
|
||||
export interface TurndownOptions {
|
||||
codeBlockStyle?: "indented" | "fenced";
|
||||
headingStyle?: "setext" | "atx";
|
||||
bulletListMarker?: "-" | "*" | "+";
|
||||
}
|
||||
|
||||
export default class TurndownService {
|
||||
constructor(options?: TurndownOptions);
|
||||
use(plugin: unknown): void;
|
||||
turndown(input: string): string;
|
||||
}
|
||||
}
|
||||
|
||||
declare module "turndown-plugin-gfm" {
|
||||
export const gfm: unknown;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
export interface Logger {
|
||||
info(message: string): void;
|
||||
warn(message: string): void;
|
||||
error(message: string): void;
|
||||
debug(message: string): void;
|
||||
}
|
||||
|
||||
export function createLogger(debugEnabled = false): Logger {
|
||||
const print = (level: string, message: string): void => {
|
||||
console.error(`[${level}] ${message}`);
|
||||
};
|
||||
|
||||
return {
|
||||
info(message: string) {
|
||||
print("info", message);
|
||||
},
|
||||
warn(message: string) {
|
||||
print("warn", message);
|
||||
},
|
||||
error(message: string) {
|
||||
print("error", message);
|
||||
},
|
||||
debug(message: string) {
|
||||
if (debugEnabled) {
|
||||
print("debug", message);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
export function normalizeUrl(input: string): URL {
|
||||
try {
|
||||
return new URL(input);
|
||||
} catch {
|
||||
throw new Error(`Invalid URL: ${input}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function sanitizeFilename(input: string): string {
|
||||
return input.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "document";
|
||||
}
|
||||
|
||||
@@ -2,7 +2,25 @@
|
||||
"name": "baoyu-url-to-markdown-scripts",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"baoyu-fetch": "./baoyu-fetch"
|
||||
},
|
||||
"scripts": {
|
||||
"reader": "bun ./lib/cli.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"baoyu-fetch": "^0.1.2"
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"chrome-launcher": "^1.2.1",
|
||||
"defuddle": "^0.17.0",
|
||||
"jsdom": "^29.0.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"turndown": "^7.2.0",
|
||||
"turndown-plugin-gfm": "^1.0.2",
|
||||
"unified": "^11.0.5",
|
||||
"ws": "^8.18.3"
|
||||
},
|
||||
"overrides": {
|
||||
"@xmldom/xmldom": "0.8.13"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user