From d6442e1ab4e188ae49c98a2066bc696e88d9986c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20Liu=20=E5=AE=9D=E7=8E=89?= Date: Tue, 21 Apr 2026 13:39:09 -0500 Subject: [PATCH] feat(baoyu-url-to-markdown): vendor baoyu-fetch runtime --- skills/baoyu-url-to-markdown/SKILL.md | 9 +- .../baoyu-url-to-markdown/scripts/baoyu-fetch | 5 + skills/baoyu-url-to-markdown/scripts/bun.lock | 94 ++- .../scripts/lib/adapters/generic/index.ts | 74 ++ .../scripts/lib/adapters/hn/index.ts | 391 +++++++++ .../scripts/lib/adapters/index.ts | 29 + .../scripts/lib/adapters/types.ts | 73 ++ .../scripts/lib/adapters/x/article.ts | 461 +++++++++++ .../scripts/lib/adapters/x/index.ts | 134 ++++ .../scripts/lib/adapters/x/login.ts | 80 ++ .../scripts/lib/adapters/x/match.ts | 9 + .../scripts/lib/adapters/x/payloads.ts | 50 ++ .../scripts/lib/adapters/x/session.ts | 47 ++ .../scripts/lib/adapters/x/shared.ts | 423 ++++++++++ .../scripts/lib/adapters/x/single.ts | 87 ++ .../scripts/lib/adapters/x/thread-loader.ts | 286 +++++++ .../scripts/lib/adapters/x/thread.ts | 316 ++++++++ .../scripts/lib/adapters/x/types.ts | 36 + .../scripts/lib/adapters/youtube/index.ts | 33 + .../lib/adapters/youtube/transcript.ts | 392 +++++++++ .../scripts/lib/adapters/youtube/utils.ts | 253 ++++++ .../scripts/lib/browser/cdp-client.ts | 258 ++++++ .../scripts/lib/browser/chrome-launcher.ts | 187 +++++ .../scripts/lib/browser/cookie-sidecar.ts | 100 +++ .../scripts/lib/browser/interaction-gates.ts | 123 +++ .../scripts/lib/browser/network-journal.ts | 235 ++++++ .../scripts/lib/browser/page-snapshot.ts | 105 +++ .../scripts/lib/browser/profile.ts | 201 +++++ .../scripts/lib/browser/session.ts | 155 ++++ .../baoyu-url-to-markdown/scripts/lib/cli.ts | 227 ++++++ .../scripts/lib/commands/convert.ts | 580 ++++++++++++++ .../scripts/lib/extract/document.ts | 51 ++ .../scripts/lib/extract/html-cleaner.ts | 467 +++++++++++ .../scripts/lib/extract/html-extractor.ts | 83 ++ .../scripts/lib/extract/html-to-markdown.ts | 758 ++++++++++++++++++ .../scripts/lib/extract/markdown-renderer.ts | 169 ++++ .../scripts/lib/media/default-downloader.ts | 161 ++++ .../scripts/lib/media/markdown-media.ts | 458 +++++++++++ .../scripts/lib/media/media-utils.ts | 261 ++++++ .../scripts/lib/media/types.ts | 34 + .../scripts/lib/types/defuddle-node.d.ts | 31 + .../scripts/lib/types/shims.d.ts | 17 + .../scripts/lib/utils/logger.ts | 30 + .../scripts/lib/utils/url.ts | 12 + .../scripts/package.json | 20 +- 45 files changed, 7960 insertions(+), 45 deletions(-) create mode 100755 skills/baoyu-url-to-markdown/scripts/baoyu-fetch create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/generic/index.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/hn/index.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/index.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/types.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/article.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/index.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/login.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/match.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/payloads.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/session.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/shared.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/single.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread-loader.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/x/types.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/index.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/transcript.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts create mode 100755 skills/baoyu-url-to-markdown/scripts/lib/cli.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/media/types.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts create mode 100644 skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts diff --git a/skills/baoyu-url-to-markdown/SKILL.md b/skills/baoyu-url-to-markdown/SKILL.md index 6914383..adad2b7 100644 --- a/skills/baoyu-url-to-markdown/SKILL.md +++ b/skills/baoyu-url-to-markdown/SKILL.md @@ -1,14 +1,13 @@ --- name: baoyu-url-to-markdown description: Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown. -version: 1.60.0 +version: 1.61.0 metadata: openclaw: homepage: https://github.com/JimLiu/baoyu-skills#baoyu-url-to-markdown requires: anyBins: - bun - - npx --- # URL to Markdown @@ -27,13 +26,13 @@ Concrete `AskUserQuestion` references below are examples — substitute the loca ## CLI Setup -**Important**: The CLI is provided by the npm package dependency `baoyu-fetch`. +**Important**: The CLI source is vendored in `{baseDir}/scripts/lib`. `scripts/package.json` installs only third-party runtime dependencies. **Agent Execution Instructions**: 1. Determine this SKILL.md file's directory path as `{baseDir}` 2. Resolve `${BUN}` runtime: if `bun` installed → `bun`; else suggest installing Bun -3. If `{baseDir}/scripts/node_modules/.bin/baoyu-fetch` does not exist, run `${BUN} install --cwd {baseDir}/scripts` -4. `${READER}` = `{baseDir}/scripts/node_modules/.bin/baoyu-fetch` +3. If `{baseDir}/scripts/node_modules` does not exist, run `${BUN} install --cwd {baseDir}/scripts` +4. `${READER}` = `{baseDir}/scripts/baoyu-fetch` 5. Replace all `${READER}` in this document with the resolved value ## Preferences (EXTEND.md) diff --git a/skills/baoyu-url-to-markdown/scripts/baoyu-fetch b/skills/baoyu-url-to-markdown/scripts/baoyu-fetch new file mode 100755 index 0000000..cd0e63b --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/baoyu-fetch @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +exec bun "$script_dir/lib/cli.ts" "$@" diff --git a/skills/baoyu-url-to-markdown/scripts/bun.lock b/skills/baoyu-url-to-markdown/scripts/bun.lock index aae0ab0..e147853 100644 --- a/skills/baoyu-url-to-markdown/scripts/bun.lock +++ b/skills/baoyu-url-to-markdown/scripts/bun.lock @@ -5,22 +5,46 @@ "": { "name": "baoyu-url-to-markdown-scripts", "dependencies": { - "baoyu-fetch": "^0.1.2", + "@mozilla/readability": "^0.6.0", + "chrome-launcher": "^1.2.1", + "defuddle": "^0.17.0", + "jsdom": "^29.0.2", + "remark-gfm": "^4.0.1", + "remark-parse": "^11.0.0", + "turndown": "^7.2.0", + "turndown-plugin-gfm": "^1.0.2", + "unified": "^11.0.5", + "ws": "^8.18.3", }, }, }, + "overrides": { + "@xmldom/xmldom": "0.8.13", + }, "packages": { - "@asamuzakjp/css-color": ["@asamuzakjp/css-color@3.2.0", "", { "dependencies": { "@csstools/css-calc": "^2.1.3", "@csstools/css-color-parser": "^3.0.9", "@csstools/css-parser-algorithms": "^3.0.4", "@csstools/css-tokenizer": "^3.0.3", "lru-cache": "^10.4.3" } }, "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw=="], + "@asamuzakjp/css-color": ["@asamuzakjp/css-color@5.1.11", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@csstools/css-calc": "^3.2.0", "@csstools/css-color-parser": "^4.1.0", "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg=="], - "@csstools/color-helpers": ["@csstools/color-helpers@5.1.0", "", {}, "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA=="], + "@asamuzakjp/dom-selector": ["@asamuzakjp/dom-selector@7.1.1", "", { "dependencies": { "@asamuzakjp/generational-cache": "^1.0.1", "@asamuzakjp/nwsapi": "^2.3.9", "bidi-js": "^1.0.3", "css-tree": "^3.2.1", "is-potential-custom-element-name": "^1.0.1" } }, "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ=="], - "@csstools/css-calc": ["@csstools/css-calc@2.1.4", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ=="], + "@asamuzakjp/generational-cache": ["@asamuzakjp/generational-cache@1.0.1", "", {}, "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg=="], - "@csstools/css-color-parser": ["@csstools/css-color-parser@3.1.0", "", { "dependencies": { "@csstools/color-helpers": "^5.1.0", "@csstools/css-calc": "^2.1.4" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA=="], + "@asamuzakjp/nwsapi": ["@asamuzakjp/nwsapi@2.3.9", "", {}, "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q=="], - "@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@3.0.5", "", { "peerDependencies": { "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ=="], + "@bramus/specificity": ["@bramus/specificity@2.4.2", "", { "dependencies": { "css-tree": "^3.0.0" }, "bin": { "specificity": "bin/cli.js" } }, "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw=="], - "@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="], + "@csstools/color-helpers": ["@csstools/color-helpers@6.0.2", "", {}, "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q=="], + + "@csstools/css-calc": ["@csstools/css-calc@3.2.0", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w=="], + + "@csstools/css-color-parser": ["@csstools/css-color-parser@4.1.0", "", { "dependencies": { "@csstools/color-helpers": "^6.0.2", "@csstools/css-calc": "^3.2.0" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ=="], + + "@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@4.0.0", "", { "peerDependencies": { "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w=="], + + "@csstools/css-syntax-patches-for-csstree": ["@csstools/css-syntax-patches-for-csstree@1.1.3", "", { "peerDependencies": { "css-tree": "^3.2.1" }, "optionalPeers": ["css-tree"] }, "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg=="], + + "@csstools/css-tokenizer": ["@csstools/css-tokenizer@4.0.0", "", {}, "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA=="], + + "@exodus/bytes": ["@exodus/bytes@1.15.0", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" }, "optionalPeers": ["@noble/hashes"] }, "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ=="], "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], @@ -38,11 +62,9 @@ "@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="], - "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], - "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], - "baoyu-fetch": ["baoyu-fetch@0.1.2", "", { "dependencies": { "@mozilla/readability": "^0.6.0", "chrome-launcher": "^1.2.1", "defuddle": "^0.14.0", "jsdom": "^26.0.0", "remark-gfm": "^4.0.1", "remark-parse": "^11.0.0", "turndown": "^7.2.0", "turndown-plugin-gfm": "^1.0.2", "unified": "^11.0.5", "ws": "^8.18.3" }, "bin": { "baoyu-fetch": "dist/cli.js" } }, "sha512-VB4CEtIcoiJo3m80RrwBuYOdC257VI6kptoI1kiRcZV78VWEicWdznRoByrjddgyHTq0WWEK5i/7L67mHcVPvQ=="], + "bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="], "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], @@ -56,13 +78,13 @@ "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + "css-tree": ["css-tree@3.2.1", "", { "dependencies": { "mdn-data": "2.27.1", "source-map-js": "^1.2.1" } }, "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA=="], + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], "cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="], - "cssstyle": ["cssstyle@4.6.0", "", { "dependencies": { "@asamuzakjp/css-color": "^3.2.0", "rrweb-cssom": "^0.8.0" } }, "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg=="], - - "data-urls": ["data-urls@5.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0" } }, "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg=="], + "data-urls": ["data-urls@7.0.0", "", { "dependencies": { "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.0" } }, "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -70,7 +92,7 @@ "decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="], - "defuddle": ["defuddle@0.14.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "linkedom": "^0.18.12", "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-btavZGd1WgiVqrVM62WGRXMUi/aU7ckTZiq0xXWLZMHvzIqNZjwIFQEDRx8MarD7fIgsB90NXZ9xHJkKtapt2Q=="], + "defuddle": ["defuddle@0.17.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "linkedom": "^0.18.12", "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-rjRb9kxgrJLFhFJjXTSAitbmmSyvjEqGCi5JiyM6ImOf4leC/O+f7TPVPAymk/gnENFTrn9j7T/LTfW9IVwiLw=="], "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], @@ -84,24 +106,18 @@ "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], - "entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + "entities": ["entities@8.0.0", "", {}, "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA=="], "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], - "html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="], + "html-encoding-sniffer": ["html-encoding-sniffer@6.0.0", "", { "dependencies": { "@exodus/bytes": "^1.6.0" } }, "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg=="], "html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="], "htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="], - "http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="], - - "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], - - "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], - "is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="], "is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], @@ -110,7 +126,7 @@ "is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="], - "jsdom": ["jsdom@26.1.0", "", { "dependencies": { "cssstyle": "^4.2.1", "data-urls": "^5.0.0", "decimal.js": "^10.5.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.16", "parse5": "^7.2.1", "rrweb-cssom": "^0.8.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^5.1.1", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.1.1", "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg=="], + "jsdom": ["jsdom@29.0.2", "", { "dependencies": { "@asamuzakjp/css-color": "^5.1.5", "@asamuzakjp/dom-selector": "^7.0.6", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.1", "@exodus/bytes": "^1.15.0", "css-tree": "^3.2.1", "data-urls": "^7.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "is-potential-custom-element-name": "^1.0.1", "lru-cache": "^11.2.7", "parse5": "^8.0.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.1", "undici": "^7.24.5", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", "whatwg-url": "^16.0.1", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w=="], "lighthouse-logger": ["lighthouse-logger@2.0.2", "", { "dependencies": { "debug": "^4.4.1", "marky": "^1.2.2" } }, "sha512-vWl2+u5jgOQuZR55Z1WM0XDdrJT6mzMP8zHUct7xTlWhuQs+eV0g+QL0RQdFjT54zVmbhLCP8vIVpy1wGn/gCg=="], @@ -118,7 +134,7 @@ "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], - "lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "lru-cache": ["lru-cache@11.3.5", "", {}, "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw=="], "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], @@ -148,6 +164,8 @@ "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="], + "mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="], + "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], @@ -208,9 +226,7 @@ "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], - "nwsapi": ["nwsapi@2.2.23", "", {}, "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ=="], - - "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + "parse5": ["parse5@8.0.1", "", { "dependencies": { "entities": "^8.0.0" } }, "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], @@ -220,23 +236,23 @@ "remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="], - "rrweb-cssom": ["rrweb-cssom@0.8.0", "", {}, "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw=="], - - "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], "saxes": ["saxes@6.0.0", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA=="], + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "symbol-tree": ["symbol-tree@3.2.4", "", {}, "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="], "temml": ["temml@0.13.2", "", {}, "sha512-n8fDRSsLscq9nh9j6z+FgkCvFMT0IJm6GCgwfzh+7AHT3Sfb4jFTQlsA6hVcF2dYYr3b66oDBVES95RfoukyrA=="], - "tldts": ["tldts@6.1.86", "", { "dependencies": { "tldts-core": "^6.1.86" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ=="], + "tldts": ["tldts@7.0.28", "", { "dependencies": { "tldts-core": "^7.0.28" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw=="], - "tldts-core": ["tldts-core@6.1.86", "", {}, "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA=="], + "tldts-core": ["tldts-core@7.0.28", "", {}, "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ=="], - "tough-cookie": ["tough-cookie@5.1.2", "", { "dependencies": { "tldts": "^6.1.32" } }, "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A=="], + "tough-cookie": ["tough-cookie@6.0.1", "", { "dependencies": { "tldts": "^7.0.5" } }, "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw=="], - "tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="], + "tr46": ["tr46@6.0.0", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw=="], "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], @@ -246,6 +262,8 @@ "uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="], + "undici": ["undici@7.25.0", "", {}, "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ=="], + "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], "unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="], @@ -264,13 +282,11 @@ "w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="], - "webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="], + "webidl-conversions": ["webidl-conversions@8.0.1", "", {}, "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ=="], - "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], + "whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="], - "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], - - "whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="], + "whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "^1.11.0", "tr46": "^6.0.0", "webidl-conversions": "^8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="], "ws": ["ws@8.20.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA=="], diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/generic/index.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/generic/index.ts new file mode 100644 index 0000000..4c417d9 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/generic/index.ts @@ -0,0 +1,74 @@ +import type { Adapter } from "../types"; +import { detectInteractionGate } from "../../browser/interaction-gates"; +import { captureNormalizedPageSnapshot } from "../../browser/page-snapshot"; +import { convertHtmlToMarkdown } from "../../extract/html-to-markdown"; + +export const genericAdapter: Adapter = { + name: "generic", + match() { + return true; + }, + async process(context) { + context.log.info(`Loading ${context.input.url.toString()} with generic adapter`); + await context.browser.goto(context.input.url.toString(), context.timeoutMs); + + try { + await context.network.waitForIdle({ + idleMs: 1_200, + timeoutMs: Math.min(context.timeoutMs, 15_000), + }); + } catch { + context.log.debug("Network idle timed out on initial load; continuing."); + } + + await context.browser.scrollToEnd({ maxSteps: 4, delayMs: 300 }); + + try { + await context.network.waitForIdle({ + idleMs: 900, + timeoutMs: Math.min(context.timeoutMs, 10_000), + }); + } catch { + context.log.debug("Network idle timed out after scrolling; continuing."); + } + + const interaction = await detectInteractionGate(context.browser); + if (interaction) { + return { + status: "needs_interaction", + interaction, + }; + } + + const snapshot = await captureNormalizedPageSnapshot(context.browser); + const converted = await convertHtmlToMarkdown(snapshot.html, snapshot.finalUrl, { + enableRemoteMarkdownFallback: context.outputFormat === "markdown", + preserveBase64Images: context.downloadMedia, + }); + const document = { + url: snapshot.finalUrl, + canonicalUrl: converted.metadata.canonicalUrl, + title: converted.metadata.title, + author: converted.metadata.author, + siteName: converted.metadata.siteName, + publishedAt: converted.metadata.publishedAt, + summary: converted.metadata.summary, + adapter: "generic", + metadata: { + coverImage: converted.metadata.coverImage, + language: converted.metadata.language, + capturedAt: converted.metadata.capturedAt, + conversionMethod: converted.conversionMethod, + fallbackReason: converted.fallbackReason, + kind: "generic/article", + }, + content: converted.markdown ? [{ type: "markdown" as const, markdown: converted.markdown }] : [], + }; + + return { + status: "ok", + document, + media: converted.media, + }; + }, +}; diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/hn/index.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/hn/index.ts new file mode 100644 index 0000000..4a37a0f --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/hn/index.ts @@ -0,0 +1,391 @@ +import { JSDOM } from "jsdom"; +import TurndownService from "turndown"; +import { gfm } from "turndown-plugin-gfm"; +import type { Adapter } from "../types"; +import type { ExtractedDocument } from "../../extract/document"; +import { collectMediaFromDocument } from "../../media/markdown-media"; + +const HN_BASE_URL = "https://news.ycombinator.com"; + +const turndown = new TurndownService({ + headingStyle: "atx", + bulletListMarker: "-", + codeBlockStyle: "fenced", +}); + +turndown.use(gfm); + +export interface HnItem { + id: number; + type: "story" | "comment" | "job" | "poll" | "pollopt" | string; + by?: string; + time?: number; + text?: string; + title?: string; + url?: string; + score?: number; + descendants?: number; + kids?: number[]; + parent?: number; + deleted?: boolean; + dead?: boolean; +} + +export interface HnCommentNode { + item: HnItem; + children: HnCommentNode[]; +} + +interface ParsedHnThread { + story: HnItem; + comments: HnCommentNode[]; +} + +function decodeHtmlText(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + + const dom = new JSDOM(`${value}`); + return dom.window.document.body.textContent?.trim() || undefined; +} + +function normalizeMarkdown(markdown: string): string { + return markdown + .replace(/\r\n/g, "\n") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function convertHnHtmlToMarkdown(html: string | undefined, baseUrl: string): string { + if (!html?.trim()) { + return ""; + } + + const dom = new JSDOM(`
${html}
`, { url: baseUrl }); + const root = dom.window.document.querySelector("#__root"); + if (!root) { + return ""; + } + + root.querySelectorAll("a[href]").forEach((element) => { + const href = element.getAttribute("href"); + if (!href) { + return; + } + + try { + element.setAttribute("href", new URL(href, baseUrl).toString()); + } catch { + // Ignore malformed URLs and keep the original href. + } + }); + + return normalizeMarkdown(turndown.turndown(root.innerHTML)); +} + +function formatIsoTimestamp(unixSeconds: number | undefined): string | undefined { + if (!unixSeconds || !Number.isFinite(unixSeconds)) { + return undefined; + } + + return new Date(unixSeconds * 1_000).toISOString(); +} + +function formatDisplayTimestamp(unixSeconds: number | undefined): string { + const iso = formatIsoTimestamp(unixSeconds); + if (!iso) { + return "unknown time"; + } + + return iso.replace("T", " ").replace(".000Z", " UTC"); +} + +function indentMarkdown(markdown: string, spaces: number): string { + const prefix = " ".repeat(spaces); + return markdown + .split("\n") + .map((line) => (line ? `${prefix}${line}` : prefix)) + .join("\n"); +} + +function renderCommentHeader(item: HnItem, pageUrl: string): string { + const author = item.by ?? "[deleted]"; + const time = item.id + ? `[${formatDisplayTimestamp(item.time)}](${pageUrl}#${item.id})` + : formatDisplayTimestamp(item.time); + return `${author} · ${time}`; +} + +function renderCommentNode(node: HnCommentNode, pageUrl: string, depth = 0): string { + const baseIndent = " ".repeat(depth * 4); + const lines = [`${baseIndent}- ${renderCommentHeader(node.item, pageUrl)}`]; + const body = convertHnHtmlToMarkdown(node.item.text, pageUrl); + + if (body) { + lines.push(""); + lines.push(indentMarkdown(body, depth * 4 + 4)); + } else if (node.item.deleted || node.item.dead) { + lines.push(""); + lines.push(`${baseIndent} [comment unavailable]`); + } + + for (const child of node.children) { + lines.push(""); + lines.push(renderCommentNode(child, pageUrl, depth + 1)); + } + + return lines.join("\n"); +} + +export function buildHnThreadMarkdown( + story: HnItem, + comments: HnCommentNode[], + pageUrl: string, +): string { + const lines: string[] = []; + const storyUrl = story.url ? new URL(story.url, pageUrl).toString() : undefined; + const storyText = convertHnHtmlToMarkdown(story.text, pageUrl); + + if (storyUrl && storyUrl !== pageUrl) { + lines.push(`Source: [${storyUrl}](${storyUrl})`); + } + lines.push(`HN Item: [${story.id}](${pageUrl})`); + + const submittedBy = story.by ? ` by ${story.by}` : ""; + const submittedAt = formatDisplayTimestamp(story.time); + lines.push(`Submitted${submittedBy} at ${submittedAt}`); + + const stats: string[] = []; + if (typeof story.score === "number") { + stats.push(`${story.score} points`); + } + if (typeof story.descendants === "number") { + stats.push(`${story.descendants} comments`); + } + if (stats.length > 0) { + lines.push(stats.join(" | ")); + } + + if (storyText) { + lines.push(""); + lines.push("## Post"); + lines.push(""); + lines.push(storyText); + } + + lines.push(""); + lines.push("## Comments"); + lines.push(""); + + if (comments.length === 0) { + lines.push("No comments."); + } else { + lines.push(comments.map((comment) => renderCommentNode(comment, pageUrl)).join("\n\n")); + } + + return normalizeMarkdown(lines.join("\n")); +} + +export function buildHnDocument( + story: HnItem, + comments: HnCommentNode[], + pageUrl: string, +): ExtractedDocument { + const decodedTitle = decodeHtmlText(story.title) ?? `HN Item ${story.id}`; + + return { + url: pageUrl, + canonicalUrl: pageUrl, + title: decodedTitle, + author: story.by, + siteName: "Hacker News", + publishedAt: formatIsoTimestamp(story.time), + adapter: "hn", + metadata: { + kind: "hn/story", + storyId: story.id, + storyUrl: story.url ? new URL(story.url, pageUrl).toString() : undefined, + points: story.score, + commentCount: story.descendants, + }, + content: [ + { + type: "markdown", + markdown: buildHnThreadMarkdown(story, comments, pageUrl), + }, + ], + }; +} + +export function parseHnItemId(url: URL): number | null { + if (url.hostname !== "news.ycombinator.com") { + return null; + } + + if (url.pathname !== "/item") { + return null; + } + + const value = url.searchParams.get("id"); + if (!value || !/^\d+$/.test(value)) { + return null; + } + + return Number(value); +} + +function extractUnixSecondsFromAge(element: Element | null): number | undefined { + const title = element?.getAttribute("title")?.trim(); + if (!title) { + return undefined; + } + + const match = title.match(/(\d{9,})$/); + return match ? Number(match[1]) : undefined; +} + +function extractScore(text: string | null | undefined): number | undefined { + if (!text) { + return undefined; + } + + const match = text.match(/(\d+)/); + return match ? Number(match[1]) : undefined; +} + +function extractCommentCount(container: ParentNode): number | undefined { + const anchors = Array.from(container.querySelectorAll("a")); + for (const anchor of anchors) { + const match = anchor.textContent?.trim().match(/(\d+)\s+comments?/i); + if (match) { + return Number(match[1]); + } + } + return undefined; +} + +function normalizeStoryUrl(storyId: number, href: string | null | undefined, pageUrl: string): string | undefined { + if (!href) { + return undefined; + } + + try { + const resolved = new URL(href, pageUrl).toString(); + if (resolved === pageUrl || resolved === `${HN_BASE_URL}/item?id=${storyId}`) { + return undefined; + } + return resolved; + } catch { + return undefined; + } +} + +export function extractHnThreadFromHtml(html: string, pageUrl: string): ParsedHnThread | null { + const dom = new JSDOM(html, { url: pageUrl }); + const { document } = dom.window; + const storyRow = document.querySelector("table.fatitem tr.athing.submission"); + if (!storyRow) { + return null; + } + + const storyId = Number(storyRow.getAttribute("id")); + if (!Number.isFinite(storyId)) { + return null; + } + + const titleLink = storyRow.querySelector(".titleline > a"); + const subline = document.querySelector("table.fatitem .subline"); + const topText = document.querySelector("table.fatitem .toptext"); + + const story: HnItem = { + id: storyId, + type: "story", + by: subline?.querySelector(".hnuser")?.textContent?.trim() || undefined, + time: extractUnixSecondsFromAge(subline?.querySelector(".age") ?? null), + title: titleLink?.innerHTML?.trim() || undefined, + url: normalizeStoryUrl(storyId, titleLink?.getAttribute("href"), pageUrl), + text: topText?.innerHTML?.trim() || undefined, + score: extractScore(subline?.querySelector(".score")?.textContent), + descendants: extractCommentCount(subline ?? document), + }; + + const roots: HnCommentNode[] = []; + const stack: HnCommentNode[] = []; + + document.querySelectorAll("tr.athing.comtr").forEach((row) => { + const commentId = Number(row.getAttribute("id")); + if (!Number.isFinite(commentId)) { + return; + } + + const indentRaw = row.querySelector("td.ind")?.getAttribute("indent"); + const depth = indentRaw && /^\d+$/.test(indentRaw) ? Number(indentRaw) : 0; + const comhead = row.querySelector(".comhead"); + const item: HnItem = { + id: commentId, + type: "comment", + by: comhead?.querySelector(".hnuser")?.textContent?.trim() || undefined, + time: extractUnixSecondsFromAge(comhead?.querySelector(".age") ?? null), + text: row.querySelector(".comment > .commtext")?.innerHTML?.trim() || undefined, + deleted: row.querySelector(".comment > .commtext") === null, + }; + + const node: HnCommentNode = { + item, + children: [], + }; + + while (stack.length > depth) { + stack.pop(); + } + + const parent = stack[stack.length - 1]; + if (parent) { + parent.children.push(node); + } else { + roots.push(node); + } + + stack.push(node); + }); + + return { + story, + comments: roots, + }; +} + +export const hnAdapter: Adapter = { + name: "hn", + match(input) { + return parseHnItemId(input.url) !== null; + }, + async process(context) { + const itemId = parseHnItemId(context.input.url); + if (!itemId) { + return { + status: "no_document", + }; + } + + const pageUrl = context.input.url.toString(); + context.log.info(`Loading ${pageUrl} with hn adapter`); + await context.browser.goto(pageUrl, context.timeoutMs); + const html = await context.browser.getHTML(); + const thread = extractHnThreadFromHtml(html, pageUrl); + if (!thread) { + return { + status: "no_document", + }; + } + + const document = buildHnDocument(thread.story, thread.comments, pageUrl); + return { + status: "ok", + document, + media: collectMediaFromDocument(document), + }; + }, +}; diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/index.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/index.ts new file mode 100644 index 0000000..25e1785 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/index.ts @@ -0,0 +1,29 @@ +import type { Adapter, AdapterInput } from "./types"; +import { genericAdapter } from "./generic"; +import { hnAdapter } from "./hn"; +import { xAdapter } from "./x"; +import { youtubeAdapter } from "./youtube"; + +const adapters: Adapter[] = [xAdapter, youtubeAdapter, hnAdapter, genericAdapter]; + +export function listAdapters(): Adapter[] { + return adapters; +} + +export function resolveAdapter(input: AdapterInput, forcedName?: string): Adapter { + if (forcedName) { + const forced = adapters.find((adapter) => adapter.name === forcedName); + if (!forced) { + throw new Error(`Unknown adapter: ${forcedName}`); + } + return forced; + } + + const matched = adapters.find((adapter) => adapter.match(input)); + if (!matched) { + throw new Error("No adapter matched the URL"); + } + return matched; +} + +export { genericAdapter }; diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/types.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/types.ts new file mode 100644 index 0000000..fa14a5c --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/types.ts @@ -0,0 +1,73 @@ +import type { BrowserSession } from "../browser/session"; +import type { CdpClient } from "../browser/cdp-client"; +import type { NetworkJournal } from "../browser/network-journal"; +import type { ExtractedDocument } from "../extract/document"; +import type { MediaDownloadRequest, MediaDownloadResult, MediaAsset } from "../media/types"; +import type { Logger } from "../utils/logger"; + +export interface AdapterInput { + url: URL; +} + +export type LoginState = "logged_in" | "logged_out" | "unknown"; +export type InteractionKind = "login" | "cloudflare" | "recaptcha" | "hcaptcha" | "captcha" | "challenge"; + +export interface AdapterLoginInfo { + provider: string; + state: LoginState; + required?: boolean; + username?: string; + reason?: string; +} + +export interface WaitForInteractionRequest { + type: "wait_for_interaction"; + kind: InteractionKind; + provider: string; + prompt: string; + reason?: string; + timeoutMs?: number; + pollIntervalMs?: number; + requiresVisibleBrowser?: boolean; +} + +export type AdapterProcessResult = + | { + status: "ok"; + document: ExtractedDocument; + media?: MediaAsset[]; + login?: AdapterLoginInfo; + } + | { + status: "needs_interaction"; + interaction: WaitForInteractionRequest; + login?: AdapterLoginInfo; + } + | { + status: "no_document"; + login?: AdapterLoginInfo; + }; + +export interface AdapterContext { + input: AdapterInput; + browser: BrowserSession; + network: NetworkJournal; + cdp: CdpClient; + log: Logger; + outputFormat: "markdown" | "json"; + timeoutMs: number; + interactive: boolean; + downloadMedia: boolean; +} + +export interface Adapter { + name: string; + match(input: AdapterInput): boolean; + checkLogin?(context: AdapterContext): Promise; + exportCookies?(context: AdapterContext, profileDir?: string): Promise; + restoreCookies?(context: AdapterContext, profileDir?: string): Promise; + downloadMedia?(request: MediaDownloadRequest): Promise; + process(context: AdapterContext): Promise; +} + +export type { MediaAsset }; diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/article.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/article.ts new file mode 100644 index 0000000..7eafa58 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/article.ts @@ -0,0 +1,461 @@ +import type { ExtractedDocument } from "../../extract/document"; +import { + findTweetNode, + findTweetNodeById, + formatMediaList, + formatTweetAuthor, + getTweetAuthorMetadata, + getTweetText, + getUser, + isRecord, + normalizeTitle, + resolveBestXVideoVariantUrl, + toHighResXImageUrl, + toXTweet, +} from "./shared"; +import type { JsonObject } from "./types"; + +interface ArticleMedia { + kind: "image" | "video"; + url: string; +} + +function resolveArticleMedia(mediaInfo: JsonObject): ArticleMedia | null { + const videoUrl = resolveBestXVideoVariantUrl(mediaInfo); + if (videoUrl) { + return { + kind: "video", + url: videoUrl, + }; + } + + const rawUrl = + (typeof mediaInfo.original_img_url === "string" && mediaInfo.original_img_url) || + (typeof mediaInfo.url === "string" && mediaInfo.url) || + ""; + + if (!rawUrl) { + return null; + } + + return { + kind: "image", + url: toHighResXImageUrl(rawUrl), + }; +} + +function resolveArticleMediaUrl(mediaInfo: JsonObject): string { + return resolveArticleMedia(mediaInfo)?.url ?? ""; +} + +function normalizeEntityMap(entityMap: unknown): Map { + const normalized = new Map(); + + if (Array.isArray(entityMap)) { + for (const entry of entityMap) { + if (!isRecord(entry)) { + continue; + } + + const key = + typeof entry.key === "string" || typeof entry.key === "number" + ? String(entry.key) + : undefined; + const value = isRecord(entry.value) ? entry.value : undefined; + if (!key || !value) { + continue; + } + normalized.set(key, value); + } + + return normalized; + } + + if (!isRecord(entityMap)) { + return normalized; + } + + for (const [key, value] of Object.entries(entityMap)) { + if (!isRecord(value)) { + continue; + } + normalized.set(key, value); + } + + return normalized; +} + +function getEntityMarkdown(entityMap: Map, entityKey: unknown): string | null { + const key = + typeof entityKey === "string" || typeof entityKey === "number" + ? String(entityKey) + : undefined; + if (!key) { + return null; + } + + const entity = entityMap.get(key); + if (!entity || entity.type !== "MARKDOWN") { + return null; + } + + const data = isRecord(entity.data) ? entity.data : {}; + if (typeof data.markdown !== "string") { + return null; + } + + const markdown = data.markdown.trim(); + return markdown || null; +} + +function getLinkUrl(entityMap: Map, entityKey: unknown): string | null { + const key = + typeof entityKey === "string" || typeof entityKey === "number" + ? String(entityKey) + : undefined; + if (!key) { + return null; + } + + const entity = entityMap.get(key); + if (!entity || entity.type !== "LINK") { + return null; + } + + const data = isRecord(entity.data) ? entity.data : {}; + const candidates = [ + data.expanded_url, + data.expandedUrl, + data.original_url, + data.originalUrl, + data.url, + data.display_url, + data.displayUrl, + ]; + + for (const candidate of candidates) { + if (typeof candidate === "string" && candidate.trim()) { + return candidate.trim(); + } + } + + return null; +} + +function getTweetId(entityMap: Map, entityKey: unknown): string | null { + const key = + typeof entityKey === "string" || typeof entityKey === "number" + ? String(entityKey) + : undefined; + if (!key) { + return null; + } + + const entity = entityMap.get(key); + if (!entity || entity.type !== "TWEET") { + return null; + } + + const data = isRecord(entity.data) ? entity.data : {}; + if (typeof data.tweetId !== "string") { + return null; + } + + return data.tweetId; +} + +function buildMediaMap(articleResult: JsonObject): Map { + const mediaMap = new Map(); + const mediaEntities = Array.isArray(articleResult.media_entities) ? articleResult.media_entities : []; + + for (const entity of mediaEntities) { + if (!isRecord(entity) || typeof entity.media_id !== "string" || !isRecord(entity.media_info)) { + continue; + } + + const media = resolveArticleMedia(entity.media_info); + if (media) { + mediaMap.set(entity.media_id, media); + } + } + + const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null; + if (coverMedia && typeof coverMedia.media_id === "string" && isRecord(coverMedia.media_info)) { + const media = resolveArticleMedia(coverMedia.media_info); + if (media) { + mediaMap.set(coverMedia.media_id, media); + } + } + + return mediaMap; +} + +function getMediaMarkdown( + entityMap: Map, + entityKey: unknown, + mediaMap: Map, +): string[] { + const key = + typeof entityKey === "string" || typeof entityKey === "number" + ? String(entityKey) + : undefined; + if (!key) { + return []; + } + + const entity = entityMap.get(key); + if (!entity || entity.type !== "MEDIA") { + return []; + } + + const data = isRecord(entity.data) ? entity.data : {}; + const mediaItems = Array.isArray(data.mediaItems) ? data.mediaItems : []; + const media: ArticleMedia[] = []; + + for (const item of mediaItems) { + if (!isRecord(item) || typeof item.mediaId !== "string") { + continue; + } + const mediaItem = mediaMap.get(item.mediaId); + if (mediaItem && !media.some((value) => value.url === mediaItem.url)) { + media.push(mediaItem); + } + } + + return media.map((item) => item.kind === "image" ? `![](${item.url})` : `[video](${item.url})`); +} + +function resolveTweetMarkdown(payloads: unknown[], tweetId: string, pageUrl: string): string | null { + for (const payload of payloads) { + const tweet = findTweetNodeById(payload, tweetId); + if (!tweet) { + continue; + } + + const xTweet = toXTweet(tweet, pageUrl); + const author = formatTweetAuthor(xTweet) ?? xTweet.url; + const lines = [`> ${author}`, ...xTweet.text.split("\n").map((line) => `> ${line}`)]; + + const media = formatMediaList(xTweet.media).map((line) => + line.startsWith("photo: ") ? `> ![](${line.slice("photo: ".length)})` : `> - ${line}`, + ); + + const parts = [lines.join("\n")]; + if (media.length > 0) { + parts.push([">", ...media].join("\n")); + } + parts.push(`> ${xTweet.url}`); + + return parts.join("\n").trim(); + } + + return `> Embedded tweet: https://x.com/i/status/${tweetId}`; +} + +function replaceLinkEntities(text: string, block: JsonObject, entityMap: Map): string { + const entityRanges = Array.isArray(block.entityRanges) ? block.entityRanges : []; + const replacements = entityRanges + .filter((range): range is JsonObject => isRecord(range)) + .map((range) => { + const offset = typeof range.offset === "number" ? range.offset : -1; + const length = typeof range.length === "number" ? range.length : -1; + const url = getLinkUrl(entityMap, range.key); + return { offset, length, url }; + }) + .filter((range) => range.offset >= 0 && range.length > 0 && range.url) + .sort((left, right) => right.offset - left.offset); + + let next = text; + for (const replacement of replacements) { + next = + next.slice(0, replacement.offset) + + replacement.url + + next.slice(replacement.offset + replacement.length); + } + return next; +} + +function renderAtomicBlock( + block: JsonObject, + entityMap: Map, + mediaMap: Map, + payloads: unknown[], + pageUrl: string, +): string | null { + const entityRanges = Array.isArray(block.entityRanges) ? block.entityRanges : []; + const parts: string[] = []; + + for (const range of entityRanges) { + if (!isRecord(range)) { + continue; + } + + const markdown = getEntityMarkdown(entityMap, range.key); + if (markdown) { + parts.push(markdown); + continue; + } + + const mediaMarkdown = getMediaMarkdown(entityMap, range.key, mediaMap); + if (mediaMarkdown.length > 0) { + parts.push(mediaMarkdown.join("\n\n")); + continue; + } + + const tweetId = getTweetId(entityMap, range.key); + if (tweetId) { + const tweetMarkdown = resolveTweetMarkdown(payloads, tweetId, pageUrl); + if (tweetMarkdown) { + parts.push(tweetMarkdown); + } + } + } + + if (parts.length === 0) { + return null; + } + + return parts.join("\n\n"); +} + +function renderArticleBlocks( + blocks: unknown[], + entityMap: Map, + mediaMap: Map, + payloads: unknown[], + pageUrl: string, +): string { + const parts: string[] = []; + let orderedCounter = 0; + + for (const block of blocks) { + if (!isRecord(block)) { + continue; + } + + const blockType = typeof block.type === "string" ? block.type : "unstyled"; + const rawText = typeof block.text === "string" ? block.text : ""; + const text = replaceLinkEntities(rawText, block, entityMap).trim(); + if (!text && blockType !== "atomic") { + continue; + } + + if (blockType !== "ordered-list-item") { + orderedCounter = 0; + } + + switch (blockType) { + case "header-one": + parts.push(`# ${text}`); + break; + case "header-two": + parts.push(`## ${text}`); + break; + case "header-three": + parts.push(`### ${text}`); + break; + case "blockquote": + parts.push(`> ${text}`); + break; + case "unordered-list-item": + parts.push(`- ${text}`); + break; + case "ordered-list-item": + orderedCounter += 1; + parts.push(`${orderedCounter}. ${text}`); + break; + case "code-block": + parts.push(`\`\`\`\n${text}\n\`\`\``); + break; + case "atomic": { + const markdown = renderAtomicBlock(block, entityMap, mediaMap, payloads, pageUrl); + if (markdown) { + parts.push(markdown); + } + break; + } + default: + parts.push(text); + break; + } + } + + return parts.join("\n\n").trim(); +} + +function getArticleResult(tweet: JsonObject): JsonObject | null { + if ( + isRecord(tweet.article) && + isRecord(tweet.article.article_results) && + isRecord(tweet.article.article_results.result) + ) { + return tweet.article.article_results.result as JsonObject; + } + return null; +} + +function extractSummary(markdown: string): string | undefined { + const segments = markdown + .split(/\n\n+/) + .map((segment) => segment.trim()) + .filter(Boolean); + + const preferred = segments.find((segment) => !/^(#|>|- |\d+\. |\`\`\`)/.test(segment)); + return preferred?.slice(0, 220); +} + +export function extractArticleDocumentFromPayload( + payload: unknown, + statusId: string, + pageUrl: string, + payloads: unknown[] = [payload], +): ExtractedDocument | null { + const tweet = findTweetNode(payload, statusId); + if (!tweet) { + return null; + } + + const articleResult = getArticleResult(tweet); + if (!articleResult) { + return null; + } + + const title = typeof articleResult.title === "string" ? articleResult.title.trim() : undefined; + const contentState = isRecord(articleResult.content_state) ? articleResult.content_state : {}; + const blocks = Array.isArray(contentState.blocks) ? contentState.blocks : []; + const entityMap = normalizeEntityMap(contentState.entityMap); + const mediaMap = buildMediaMap(articleResult); + const richMarkdown = renderArticleBlocks(blocks, entityMap, mediaMap, payloads, pageUrl); + const plainText = typeof articleResult.plain_text === "string" ? articleResult.plain_text.trim() : ""; + const markdown = richMarkdown || plainText || getTweetText(tweet); + if (!markdown) { + return null; + } + + const xTweet = toXTweet(tweet, pageUrl); + const user = getUser(tweet); + const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null; + const coverMediaInfo = coverMedia && isRecord(coverMedia.media_info) ? coverMedia.media_info : null; + const coverImage = coverMediaInfo ? resolveArticleMediaUrl(coverMediaInfo) || undefined : undefined; + + return { + url: pageUrl, + canonicalUrl: xTweet.url, + title: title || normalizeTitle(xTweet.text, "X Article"), + author: formatTweetAuthor(xTweet), + siteName: "X", + publishedAt: xTweet.createdAt, + summary: extractSummary(markdown) || xTweet.text.slice(0, 200) || undefined, + adapter: "x", + metadata: { + kind: "x/article", + tweetId: xTweet.id, + coverImage, + authorName: xTweet.authorName ?? user.name, + authorUsername: xTweet.author ?? user.screenName, + authorUrl: (xTweet.author ?? user.screenName) ? `https://x.com/${xTweet.author ?? user.screenName}` : undefined, + ...getTweetAuthorMetadata(xTweet), + }, + content: [{ type: "markdown", markdown }], + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/index.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/index.ts new file mode 100644 index 0000000..f495c08 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/index.ts @@ -0,0 +1,134 @@ +import type { Adapter, AdapterLoginInfo } from "../types"; +import { exportCookies, restoreCookies, type CookieSidecarConfig } from "../../browser/cookie-sidecar"; +import { detectInteractionGate } from "../../browser/interaction-gates"; +import type { ExtractedDocument } from "../../extract/document"; +import { collectMediaFromDocument } from "../../media/markdown-media"; +import { extractArticleDocumentFromPayload } from "./article"; +import { buildNeedsLoginResult, detectXLogin } from "./login"; +import { extractStatusId, isXHost } from "./match"; +import { collectXJsonPayloads, waitForInitialXPayload } from "./payloads"; +import { extractSingleTweetDocumentFromPayload } from "./single"; +import { extractThreadDocumentFromPayloads } from "./thread"; +import { loadFullXThread } from "./thread-loader"; + +const cookieConfig: CookieSidecarConfig = { + urls: ["https://x.com/", "https://twitter.com/"], + filename: "x-session-cookies.json", + requiredCookieNames: ["auth_token", "ct0"], + filterCookie: (c) => { + const d = c.domain ?? ""; + return d.endsWith("x.com") || d.endsWith("twitter.com"); + }, +}; + +function extractDocumentFromPayloads( + payloads: unknown[], + statusId: string, + pageUrl: string, +): ExtractedDocument | null { + for (const payload of payloads) { + const articleDocument = extractArticleDocumentFromPayload(payload, statusId, pageUrl, payloads); + if (articleDocument) { + return articleDocument; + } + } + + const threadDocument = extractThreadDocumentFromPayloads(payloads, statusId, pageUrl); + if (threadDocument) { + return threadDocument; + } + + for (const payload of payloads) { + const singleDocument = extractSingleTweetDocumentFromPayload(payload, statusId, pageUrl); + if (singleDocument) { + return singleDocument; + } + } + + return null; +} + +async function ensureXLoginState(context: Parameters[0]): Promise { + return detectXLogin(context); +} + +export const xAdapter: Adapter = { + name: "x", + match(input) { + return isXHost(input.url.hostname); + }, + async checkLogin(context) { + return detectXLogin(context); + }, + async exportCookies(context, profileDir) { + return exportCookies(context.browser.targetSession, cookieConfig, profileDir); + }, + async restoreCookies(context, profileDir) { + return restoreCookies(context.browser.targetSession, cookieConfig, profileDir); + }, + async process(context) { + const statusId = extractStatusId(context.input.url); + if (!statusId) { + return { + status: "no_document", + }; + } + + context.log.info(`Loading ${context.input.url.toString()} with x adapter`); + await context.browser.goto(context.input.url.toString(), context.timeoutMs); + + const interaction = await detectInteractionGate(context.browser); + if (interaction) { + return { + status: "needs_interaction", + interaction, + }; + } + + let login = await ensureXLoginState(context); + if (login.state === "logged_out") { + return buildNeedsLoginResult(login); + } + + await waitForInitialXPayload(context); + await loadFullXThread(context, statusId); + + const pageUrl = await context.browser.getURL(); + const postLoadInteraction = await detectInteractionGate(context.browser); + if (postLoadInteraction) { + return { + status: "needs_interaction", + interaction: postLoadInteraction, + login, + }; + } + + login = await ensureXLoginState(context).catch(() => login); + if (login.state === "logged_out") { + return buildNeedsLoginResult(login); + } + + const payloads = await collectXJsonPayloads(context); + if (payloads.length === 0) { + return { + status: "no_document", + login, + }; + } + + const document = extractDocumentFromPayloads(payloads, statusId, pageUrl); + if (document) { + return { + status: "ok", + document, + media: collectMediaFromDocument(document), + login, + }; + } + + return { + status: "no_document", + login, + }; + }, +}; diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/login.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/login.ts new file mode 100644 index 0000000..ee3f250 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/login.ts @@ -0,0 +1,80 @@ +import type { AdapterContext, AdapterLoginInfo, AdapterProcessResult } from "../types"; + +interface XLoginSnapshot { + currentUrl: string; + hasAccountMenu: boolean; + hasLoginInputs: boolean; + bodyText: string; +} + +export async function detectXLogin(context: AdapterContext): Promise { + const snapshot = await context.browser.evaluate(` + (() => { + const bodyText = (document.body?.innerText ?? "").slice(0, 2500); + return { + currentUrl: window.location.href, + hasAccountMenu: Boolean( + document.querySelector( + '[data-testid="SideNav_AccountSwitcher_Button"], [data-testid="AppTabBar_Profile_Link"], [aria-label="Account menu"]' + ) + ), + hasLoginInputs: Boolean( + document.querySelector( + 'input[name="text"], input[name="password"], input[autocomplete="username"], input[autocomplete="current-password"]' + ) + ), + bodyText, + }; + })() + `).catch(async () => ({ + currentUrl: await context.browser.getURL().catch(() => context.input.url.toString()), + hasAccountMenu: false, + hasLoginInputs: false, + bodyText: "", + })); + + if ( + /\/i\/flow\/login|\/login/i.test(snapshot.currentUrl) || + snapshot.hasLoginInputs || + /sign in to x|join x today|登录 x|注册 x|登录到 x/i.test(snapshot.bodyText) + ) { + return { + provider: "x", + state: "logged_out", + required: true, + reason: "X login page detected", + }; + } + + if (snapshot.hasAccountMenu) { + return { + provider: "x", + state: "logged_in", + }; + } + + return { + provider: "x", + state: "unknown", + }; +} + +export function buildNeedsLoginResult(login: AdapterLoginInfo): AdapterProcessResult { + return { + status: "needs_interaction", + login: { + ...login, + provider: "x", + state: login.state === "logged_in" ? "unknown" : login.state, + required: true, + }, + interaction: { + type: "wait_for_interaction", + kind: "login", + provider: "x", + reason: login.reason, + prompt: "Please sign in to X in the opened Chrome window. Extraction will continue automatically once login is detected.", + requiresVisibleBrowser: true, + }, + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/match.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/match.ts new file mode 100644 index 0000000..641974b --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/match.ts @@ -0,0 +1,9 @@ +export function isXHost(hostname: string): boolean { + return ["x.com", "www.x.com", "twitter.com", "www.twitter.com"].includes(hostname); +} + +export function extractStatusId(url: URL): string | undefined { + const match = url.pathname.match(/\/(?:status|article)\/(\d+)/); + return match?.[1]; +} + diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/payloads.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/payloads.ts new file mode 100644 index 0000000..ff85288 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/payloads.ts @@ -0,0 +1,50 @@ +import type { AdapterContext } from "../types"; +import { filterXGraphQlEntries } from "./shared"; + +export function getRelevantXThreadEntries(context: AdapterContext) { + return filterXGraphQlEntries(context.network.getEntries()).filter( + (entry) => + entry.method === "GET" && + entry.finished && + ( + entry.url.includes("TweetDetail") || + entry.url.includes("TweetResultByRestId") || + entry.url.includes("TweetResultsByRestIds") + ), + ); +} + +export async function prefetchRelevantXThreadBodies(context: AdapterContext): Promise { + const entries = getRelevantXThreadEntries(context).filter((entry) => entry.body === undefined && !entry.bodyError); + for (const entry of entries) { + await context.network.ensureBody(entry); + } +} + +export async function collectXJsonPayloads(context: AdapterContext): Promise { + await prefetchRelevantXThreadBodies(context); + const entries = getRelevantXThreadEntries(context); + + const payloads: unknown[] = []; + for (const entry of entries) { + const payload = await context.network.getJsonBody(entry); + if (payload) { + payloads.push(payload); + } + } + return payloads; +} + +export async function waitForInitialXPayload(context: AdapterContext): Promise { + try { + await context.network.waitForResponse( + (entry) => + entry.url.includes("/graphql/") && + (entry.url.includes("TweetDetail") || entry.url.includes("TweetResultByRestId")), + { timeoutMs: Math.min(context.timeoutMs, 15_000) }, + ); + await prefetchRelevantXThreadBodies(context); + } catch { + context.log.debug("No tweet GraphQL response observed before timeout."); + } +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/session.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/session.ts new file mode 100644 index 0000000..54b2ba0 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/session.ts @@ -0,0 +1,47 @@ +import type { AdapterContext } from "../types"; + +const X_SESSION_URLS = ["https://x.com/", "https://twitter.com/"] as const; +const REQUIRED_X_SESSION_COOKIES = ["auth_token", "ct0"] as const; + +interface CookieLike { + name?: string; + value?: string | null; +} + +interface NetworkGetCookiesResult { + cookies?: CookieLike[]; +} + +export function buildXSessionCookieMap(cookies: readonly CookieLike[]): Record { + const cookieMap: Record = {}; + for (const cookie of cookies) { + const name = cookie.name?.trim(); + const value = cookie.value?.trim(); + if (!name || !value) { + continue; + } + cookieMap[name] = value; + } + return cookieMap; +} + +export function hasRequiredXSessionCookies(cookieMap: Record): boolean { + return REQUIRED_X_SESSION_COOKIES.every((name) => Boolean(cookieMap[name])); +} + +export async function readXSessionCookieMap( + context: Pick, +): Promise> { + const { cookies } = await context.browser.targetSession.send( + "Network.getCookies", + { urls: [...X_SESSION_URLS] }, + ); + return buildXSessionCookieMap(cookies ?? []); +} + +export async function isXSessionReady( + context: Pick, +): Promise { + const cookieMap = await readXSessionCookieMap(context); + return hasRequiredXSessionCookies(cookieMap); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/shared.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/shared.ts new file mode 100644 index 0000000..a0fef18 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/shared.ts @@ -0,0 +1,423 @@ +import path from "node:path"; +import type { NetworkEntry } from "../../browser/network-journal"; +import type { XMedia, XQuotedTweet, XTweet, XUser, JsonObject } from "./types"; + +const X_IMAGE_EXTENSIONS = new Set(["jpg", "jpeg", "png", "webp", "gif", "bmp", "avif"]); + +function emptyObject(): JsonObject { + return {}; +} + +export function isRecord(value: unknown): value is JsonObject { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +export function walk(value: unknown, visitor: (node: unknown) => boolean | void): boolean { + if (visitor(value)) { + return true; + } + + if (Array.isArray(value)) { + for (const item of value) { + if (walk(item, visitor)) { + return true; + } + } + return false; + } + + if (isRecord(value)) { + for (const child of Object.values(value)) { + if (walk(child, visitor)) { + return true; + } + } + } + + return false; +} + +function hasTweetText(node: JsonObject): boolean { + const legacy = isRecord(node.legacy) ? node.legacy : emptyObject(); + return ( + typeof legacy.full_text === "string" || + typeof getNoteTweetText(node) === "string" + ); +} + +export function findTweetNodeById(payload: unknown, tweetId: string): JsonObject | null { + let match: JsonObject | null = null; + + walk(payload, (node) => { + if (!isRecord(node) || typeof node.rest_id !== "string" || !isRecord(node.legacy)) { + return false; + } + + if (!hasTweetText(node)) { + return false; + } + + if (node.rest_id === tweetId) { + match = node; + return true; + } + + return false; + }); + + return match; +} + +export function findTweetNode(payload: unknown, statusId: string): JsonObject | null { + let firstMatch: JsonObject | null = null; + const exactMatch = findTweetNodeById(payload, statusId); + if (exactMatch) { + return exactMatch; + } + + walk(payload, (node) => { + if (!isRecord(node) || typeof node.rest_id !== "string" || !isRecord(node.legacy)) { + return false; + } + if (!hasTweetText(node)) { + return false; + } + if (!firstMatch) { + firstMatch = node; + } + return false; + }); + + return firstMatch; +} + +export function getLegacy(tweet: JsonObject): JsonObject { + return isRecord(tweet.legacy) ? tweet.legacy : emptyObject(); +} + +export function unwrapTweetResult(node: unknown): JsonObject | null { + if (!isRecord(node)) { + return null; + } + + if (node.__typename === "TweetWithVisibilityResults" && isRecord(node.tweet)) { + return unwrapTweetResult(node.tweet); + } + + const tweet = isRecord(node.tweet) ? (node.tweet as JsonObject) : node; + if (typeof tweet.rest_id !== "string" || !isRecord(tweet.legacy)) { + return null; + } + + return tweet; +} + +export function getUser(tweet: JsonObject): XUser { + const result = + isRecord(tweet.core) && + isRecord(tweet.core.user_results) && + isRecord(tweet.core.user_results.result) + ? (tweet.core.user_results.result as JsonObject) + : emptyObject(); + const legacy = isRecord(result.legacy) ? result.legacy : emptyObject(); + const core = isRecord(result.core) ? result.core : emptyObject(); + return { + name: + (typeof legacy.name === "string" ? legacy.name : undefined) ?? + (typeof core.name === "string" ? core.name : undefined), + screenName: + (typeof legacy.screen_name === "string" ? legacy.screen_name : undefined) ?? + (typeof core.screen_name === "string" ? core.screen_name : undefined), + }; +} + +function getNoteTweetResult(tweet: JsonObject): JsonObject | null { + if ( + !isRecord(tweet.note_tweet) || + !isRecord(tweet.note_tweet.note_tweet_results) || + !isRecord(tweet.note_tweet.note_tweet_results.result) + ) { + return null; + } + + return tweet.note_tweet.note_tweet_results.result as JsonObject; +} + +function getNoteTweetText(tweet: JsonObject): string | undefined { + const noteTweet = getNoteTweetResult(tweet); + return typeof noteTweet?.text === "string" ? noteTweet.text : undefined; +} + +interface TweetUrlEntity { + url: string; + expandedUrl?: string; + displayUrl?: string; +} + +function collectTweetUrlEntities(values: unknown[]): TweetUrlEntity[] { + return values.reduce((entities, value) => { + if (!isRecord(value) || typeof value.url !== "string" || !value.url) { + return entities; + } + + entities.push({ + url: value.url, + expandedUrl: typeof value.expanded_url === "string" ? value.expanded_url : undefined, + displayUrl: typeof value.display_url === "string" ? value.display_url : undefined, + }); + + return entities; + }, []); +} + +function getTweetUrlEntities(tweet: JsonObject): TweetUrlEntity[] { + const noteTweet = getNoteTweetResult(tweet); + const noteTweetEntitySet = noteTweet && isRecord(noteTweet.entity_set) ? noteTweet.entity_set : emptyObject(); + const noteTweetUrls = collectTweetUrlEntities(Array.isArray(noteTweetEntitySet.urls) ? noteTweetEntitySet.urls : []); + + const legacy = getLegacy(tweet); + const legacyEntities = isRecord(legacy.entities) ? legacy.entities : emptyObject(); + const legacyUrls = collectTweetUrlEntities(Array.isArray(legacyEntities.urls) ? legacyEntities.urls : []); + + const seen = new Set(); + return [...noteTweetUrls, ...legacyUrls].filter((value) => { + if (seen.has(value.url)) { + return false; + } + seen.add(value.url); + return true; + }); +} + +export function getTweetText(tweet: JsonObject): string { + const legacy = getLegacy(tweet); + let text = + getNoteTweetText(tweet) ?? (typeof legacy.full_text === "string" ? legacy.full_text : ""); + + for (const value of getTweetUrlEntities(tweet)) { + const replacement = + (typeof value.expandedUrl === "string" && value.expandedUrl) || + (typeof value.displayUrl === "string" && value.displayUrl) || + value.url; + text = text.replaceAll(value.url, replacement); + } + + const extendedEntities = isRecord(legacy.extended_entities) ? legacy.extended_entities : emptyObject(); + const media = Array.isArray(extendedEntities.media) ? extendedEntities.media : []; + for (const value of media) { + if (isRecord(value) && typeof value.url === "string") { + text = text.replaceAll(value.url, "").trim(); + } + } + + return text.replace(/\n{3,}/g, "\n\n").trim(); +} + +function normalizeXImageExtension(raw: string | undefined | null): string | undefined { + if (!raw) { + return undefined; + } + + const normalized = raw.replace(/^\./, "").trim().toLowerCase(); + if (!normalized) { + return undefined; + } + + return normalized === "jpeg" ? "jpg" : normalized; +} + +export function toHighResXImageUrl(rawUrl: string): string { + try { + const parsed = new URL(rawUrl); + if (parsed.hostname.toLowerCase() !== "pbs.twimg.com") { + return rawUrl; + } + + const pathExtension = normalizeXImageExtension(path.posix.extname(parsed.pathname)); + const format = normalizeXImageExtension(parsed.searchParams.get("format")) ?? pathExtension; + if (!format || !X_IMAGE_EXTENSIONS.has(format)) { + return rawUrl; + } + + if (pathExtension) { + parsed.pathname = parsed.pathname.replace(new RegExp(`\\.${pathExtension}$`, "i"), ""); + } + + parsed.searchParams.set("format", format); + parsed.searchParams.set("name", "4096x4096"); + return parsed.toString(); + } catch { + return rawUrl; + } +} + +function getVideoVariantBitrate(variant: JsonObject): number { + const value = variant.bitrate ?? variant.bit_rate; + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function getVideoVariantContentType(variant: JsonObject): string { + const value = variant.content_type ?? variant.contentType; + return typeof value === "string" ? value.toLowerCase() : ""; +} + +export function resolveBestXVideoVariantUrl(mediaInfo: unknown): string | undefined { + if (!isRecord(mediaInfo)) { + return undefined; + } + + const variantsSource = + Array.isArray(mediaInfo.variants) + ? mediaInfo.variants + : isRecord(mediaInfo.video_info) && Array.isArray(mediaInfo.video_info.variants) + ? mediaInfo.video_info.variants + : []; + + const variants = variantsSource + .filter( + (variant): variant is JsonObject => + isRecord(variant) && typeof variant.url === "string" && variant.url.length > 0, + ) + .filter((variant) => getVideoVariantContentType(variant) === "video/mp4") + .sort((left, right) => getVideoVariantBitrate(right) - getVideoVariantBitrate(left)); + + return typeof variants[0]?.url === "string" ? variants[0].url : undefined; +} + +export function getTweetMedia(tweet: JsonObject): XMedia[] { + const legacy = getLegacy(tweet); + const extendedEntities = isRecord(legacy.extended_entities) ? legacy.extended_entities : emptyObject(); + const media = Array.isArray(extendedEntities.media) ? extendedEntities.media : []; + + return media + .map((value) => { + if (!isRecord(value) || typeof value.type !== "string") { + return null; + } + if (value.type === "photo" && typeof value.media_url_https === "string") { + return { + type: value.type, + url: toHighResXImageUrl(value.media_url_https), + alt: typeof value.ext_alt_text === "string" ? value.ext_alt_text : undefined, + }; + } + if (value.type === "video" || value.type === "animated_gif") { + const videoUrl = resolveBestXVideoVariantUrl(value); + if (!videoUrl) { + return null; + } + return { + type: value.type, + url: videoUrl, + }; + } + return null; + }) + .filter((value): value is XMedia => value !== null); +} + +export function getTweetUrl(tweet: JsonObject, fallbackUrl: string): string { + const user = getUser(tweet); + const fallbackScreenName = extractScreenNameFromUrl(fallbackUrl); + const id = typeof tweet.rest_id === "string" ? tweet.rest_id : ""; + const screenName = user.screenName ?? fallbackScreenName; + if (screenName && id) { + return `https://x.com/${screenName}/status/${id}`; + } + return fallbackUrl; +} + +export function getQuotedTweet(tweet: JsonObject, fallbackUrl: string): XQuotedTweet | undefined { + const quoted = unwrapTweetResult( + isRecord(tweet.quoted_status_result) ? tweet.quoted_status_result.result : null, + ); + if (!quoted) { + return undefined; + } + + const user = getUser(quoted); + return { + id: typeof quoted.rest_id === "string" ? quoted.rest_id : "", + author: user.screenName, + authorName: user.name, + text: getTweetText(quoted), + url: getTweetUrl(quoted, fallbackUrl), + media: getTweetMedia(quoted), + }; +} + +export function extractScreenNameFromUrl(url: string): string | undefined { + try { + const parsed = new URL(url); + const match = parsed.pathname.match(/^\/([^/]+)\/(?:status|article)\//); + if (!match) { + return undefined; + } + if (match[1] === "i") { + return undefined; + } + return match[1]; + } catch { + return undefined; + } +} + +export function toXTweet(tweet: JsonObject, fallbackUrl: string): XTweet { + const legacy = getLegacy(tweet); + const user = getUser(tweet); + const fallbackScreenName = extractScreenNameFromUrl(fallbackUrl); + const screenName = user.screenName ?? fallbackScreenName; + return { + id: typeof tweet.rest_id === "string" ? tweet.rest_id : "", + author: screenName, + authorName: user.name, + text: getTweetText(tweet), + likes: typeof legacy.favorite_count === "number" ? legacy.favorite_count : 0, + retweets: typeof legacy.retweet_count === "number" ? legacy.retweet_count : 0, + replies: typeof legacy.reply_count === "number" ? legacy.reply_count : 0, + createdAt: typeof legacy.created_at === "string" ? legacy.created_at : undefined, + inReplyTo: typeof legacy.in_reply_to_status_id_str === "string" ? legacy.in_reply_to_status_id_str : undefined, + url: getTweetUrl(tweet, fallbackUrl), + media: getTweetMedia(tweet), + quotedTweet: getQuotedTweet(tweet, fallbackUrl), + }; +} + +export function normalizeTitle(text: string, fallback: string): string { + const firstLine = text.split("\n")[0]?.trim(); + if (!firstLine) { + return fallback; + } + return firstLine.slice(0, 120); +} + +export function formatTweetAuthor(tweet: XTweet): string | undefined { + if (tweet.author && tweet.authorName) { + return `${tweet.authorName} (@${tweet.author})`; + } + if (tweet.author) { + return `@${tweet.author}`; + } + return tweet.authorName; +} + +export function getTweetAuthorMetadata(tweet: XTweet): Record { + return { + authorName: tweet.authorName, + authorUsername: tweet.author, + authorUrl: tweet.author ? `https://x.com/${tweet.author}` : undefined, + }; +} + +export function formatMediaList(media: XMedia[]): string[] { + return media.map((item) => { + if (item.type === "photo") { + return `photo: ${item.url}`; + } + return `${item.type}: ${item.url}`; + }); +} + +export function filterXGraphQlEntries(entries: NetworkEntry[]): NetworkEntry[] { + return entries.filter((entry) => entry.url.includes("/graphql/")); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/single.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/single.ts new file mode 100644 index 0000000..aecceb8 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/single.ts @@ -0,0 +1,87 @@ +import type { ExtractedDocument, ContentBlock } from "../../extract/document"; +import { findTweetNode, formatMediaList, formatTweetAuthor, getTweetAuthorMetadata, normalizeTitle, toXTweet } from "./shared"; + +export function extractSingleTweetDocumentFromPayload( + payload: unknown, + statusId: string, + pageUrl: string, +): ExtractedDocument | null { + const tweet = findTweetNode(payload, statusId); + if (!tweet) { + return null; + } + + const xTweet = toXTweet(tweet, pageUrl); + const content: ContentBlock[] = []; + + if (xTweet.text) { + content.push({ type: "paragraph", text: xTweet.text }); + } + + for (const mediaLine of formatMediaList(xTweet.media)) { + if (mediaLine.startsWith("photo: ")) { + content.push({ + type: "image", + url: mediaLine.slice("photo: ".length), + }); + } else { + content.push({ + type: "list", + ordered: false, + items: [mediaLine], + }); + } + } + + if (xTweet.quotedTweet) { + const quotedLines: string[] = []; + const quotedAuthor = + xTweet.quotedTweet.author && xTweet.quotedTweet.authorName + ? `${xTweet.quotedTweet.authorName} (@${xTweet.quotedTweet.author})` + : xTweet.quotedTweet.author + ? `@${xTweet.quotedTweet.author}` + : xTweet.quotedTweet.authorName; + + if (quotedAuthor) { + quotedLines.push(quotedAuthor); + } + if (xTweet.quotedTweet.text) { + quotedLines.push(xTweet.quotedTweet.text); + } + quotedLines.push(...formatMediaList(xTweet.quotedTweet.media)); + + if (quotedLines.length > 0) { + content.push({ type: "heading", depth: 2, text: "Quoted Tweet" }); + content.push({ type: "quote", text: quotedLines.join("\n\n") }); + } + } + + return { + url: pageUrl, + canonicalUrl: xTweet.url, + title: normalizeTitle( + xTweet.author ? `@${xTweet.author}: ${xTweet.text}` : xTweet.text, + "Tweet", + ), + author: formatTweetAuthor(xTweet), + siteName: "X", + publishedAt: xTweet.createdAt, + summary: xTweet.text.slice(0, 200) || undefined, + adapter: "x", + metadata: { + kind: "x/post", + tweetId: xTweet.id, + ...getTweetAuthorMetadata(xTweet), + conversationId: + typeof tweet.legacy === "object" && + tweet.legacy !== null && + typeof (tweet.legacy as Record).conversation_id_str === "string" + ? (tweet.legacy as Record).conversation_id_str + : undefined, + favoriteCount: xTweet.likes, + replyCount: xTweet.replies, + retweetCount: xTweet.retweets, + }, + content, + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread-loader.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread-loader.ts new file mode 100644 index 0000000..e4ee843 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread-loader.ts @@ -0,0 +1,286 @@ +import type { AdapterContext } from "../types"; +import { extractThreadTweetsFromPayloads } from "./thread"; +import { collectXJsonPayloads, getRelevantXThreadEntries, prefetchRelevantXThreadBodies } from "./payloads"; + +interface ClickTextResult { + clicked: boolean; + text?: string; +} + +interface ScrollStepResult { + moved: boolean; + atTop: boolean; + atBottom: boolean; +} + +interface ThreadProgress { + tweetCount: number; + firstTweetId?: string; + lastTweetId?: string; + requestCount: number; + tweetDetailCount: number; +} + +interface TopProbeState { + requestCount: number; + tweetDetailCount: number; + scrollHeight: number; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitForXNetworkSettle(context: AdapterContext, reason: string): Promise { + try { + await context.network.waitForIdle({ + idleMs: 650, + timeoutMs: Math.min(context.timeoutMs, 5_000), + }); + } catch { + context.log.debug(`Network idle timed out after ${reason}.`); + } +} + +async function captureTopProbeState(context: AdapterContext): Promise { + const entries = getRelevantXThreadEntries(context); + const scrollHeight = await context.browser.evaluate(` + (() => { + const scrollRoot = document.scrollingElement ?? document.documentElement ?? document.body; + return scrollRoot.scrollHeight; + })() + `); + + return { + requestCount: entries.length, + tweetDetailCount: entries.filter((entry) => entry.url.includes("TweetDetail")).length, + scrollHeight, + }; +} + +async function waitForTopProbe(context: AdapterContext): Promise { + const initial = await captureTopProbeState(context); + const deadline = Date.now() + 1_200; + + while (Date.now() < deadline) { + try { + await context.network.waitForIdle({ + idleMs: 250, + timeoutMs: 350, + }); + } catch { + // Keep polling until the shorter top-probe budget expires. + } + + await prefetchRelevantXThreadBodies(context); + const next = await captureTopProbeState(context); + if ( + next.requestCount > initial.requestCount || + next.tweetDetailCount > initial.tweetDetailCount || + next.scrollHeight > initial.scrollHeight + 4 + ) { + context.log.debug("Observed additional X thread activity while probing the page top."); + return true; + } + + await sleep(120); + } + + return false; +} + +async function scrollThreadToTop(context: AdapterContext): Promise { + let settledTopChecks = 0; + + while (settledTopChecks < 2) { + const scroll = await context.browser.evaluate(` + (() => { + const scrollRoot = document.scrollingElement ?? document.documentElement ?? document.body; + const before = window.scrollY; + window.scrollTo({ top: 0, left: 0, behavior: "instant" }); + const after = window.scrollY; + return { + moved: after !== before, + atTop: after <= 4, + atBottom: window.innerHeight + after >= scrollRoot.scrollHeight - 4, + }; + })() + `); + await sleep(140); + await waitForXNetworkSettle(context, "scrolling X thread to top"); + await prefetchRelevantXThreadBodies(context); + + if (scroll.moved) { + settledTopChecks = 0; + continue; + } + + const observedTopActivity = await waitForTopProbe(context); + if (observedTopActivity) { + settledTopChecks = 0; + continue; + } + + settledTopChecks += 1; + } +} + +async function clickVisibleShowReplies(context: AdapterContext): Promise { + return context.browser.evaluate(` + (() => { + const normalize = (value) => value.replace(/\\s+/g, " ").trim(); + const matches = [ + /^Show replies$/i, + /^Show more replies$/i, + /^Show additional replies$/i, + /^显示回复$/, + /^展开回复$/, + ]; + const isVisible = (element) => { + if (!(element instanceof HTMLElement)) { + return false; + } + const rect = element.getBoundingClientRect(); + const style = window.getComputedStyle(element); + return ( + rect.width > 0 && + rect.height > 0 && + style.visibility !== "hidden" && + style.display !== "none" + ); + }; + + const selectors = [ + "a", + "button", + '[role="button"]', + '[role="link"]', + ]; + + for (const element of document.querySelectorAll(selectors.join(","))) { + if (!isVisible(element)) { + continue; + } + const text = normalize(element.textContent ?? ""); + if (!text || !matches.some((pattern) => pattern.test(text))) { + continue; + } + element.scrollIntoView({ block: "center", inline: "nearest" }); + if (element instanceof HTMLElement) { + element.click(); + return { clicked: true, text }; + } + } + + return { clicked: false }; + })() + `); +} + +async function expandVisibleShowReplies(context: AdapterContext): Promise { + let clickCount = 0; + + while (clickCount < 8) { + const result = await clickVisibleShowReplies(context).catch(() => ({ clicked: false })); + if (!result.clicked) { + break; + } + + clickCount += 1; + context.log.debug(`Expanded X thread replies via "${result.text ?? "Show replies"}".`); + await sleep(250); + await waitForXNetworkSettle(context, "expanding Show replies"); + await prefetchRelevantXThreadBodies(context); + } + + return clickCount; +} + +async function scrollThreadBy(context: AdapterContext, stepPx: number): Promise { + const result = await context.browser.evaluate(` + (() => { + const scrollRoot = document.scrollingElement ?? document.documentElement ?? document.body; + const before = window.scrollY; + window.scrollBy({ top: ${stepPx}, left: 0, behavior: "instant" }); + const after = window.scrollY; + return { + moved: after !== before, + atTop: after <= 4, + atBottom: window.innerHeight + after >= scrollRoot.scrollHeight - 4, + }; + })() + `); + + await sleep(140); + await waitForXNetworkSettle(context, "scrolling X thread"); + await prefetchRelevantXThreadBodies(context); + return result; +} + +async function captureThreadProgress(context: AdapterContext, statusId: string): Promise { + const entries = getRelevantXThreadEntries(context); + const payloads = await collectXJsonPayloads(context); + const tweets = extractThreadTweetsFromPayloads(payloads, statusId, context.input.url.toString()); + return { + tweetCount: tweets.length, + firstTweetId: tweets[0]?.id, + lastTweetId: tweets[tweets.length - 1]?.id, + requestCount: entries.length, + tweetDetailCount: entries.filter((entry) => entry.url.includes("TweetDetail")).length, + }; +} + +export async function loadFullXThread(context: AdapterContext, statusId: string): Promise { + await scrollThreadToTop(context); + + let progress = await captureThreadProgress(context, statusId); + let stagnantRounds = 0; + let roundsWithoutMovement = 0; + let distanceWithoutThreadActivityPx = 0; + + for (let round = 0; ; round += 1) { + const stepPx = round < 12 ? 1_200 : 1_600; + let expandedCount = await expandVisibleShowReplies(context); + const scroll = await scrollThreadBy(context, stepPx); + expandedCount += await expandVisibleShowReplies(context); + const nextProgress = await captureThreadProgress(context, statusId); + const grew = + nextProgress.tweetCount > progress.tweetCount || + nextProgress.firstTweetId !== progress.firstTweetId || + nextProgress.lastTweetId !== progress.lastTweetId || + nextProgress.requestCount > progress.requestCount || + nextProgress.tweetDetailCount > progress.tweetDetailCount; + + if (grew) { + context.log.debug( + `X thread progress: ${nextProgress.tweetCount} tweets (${nextProgress.firstTweetId ?? "unknown"} -> ${nextProgress.lastTweetId ?? "unknown"}), ${nextProgress.requestCount} requests, ${nextProgress.tweetDetailCount} TweetDetail.`, + ); + stagnantRounds = 0; + distanceWithoutThreadActivityPx = 0; + } else if (expandedCount > 0) { + stagnantRounds = 0; + distanceWithoutThreadActivityPx = 0; + } else { + stagnantRounds += 1; + distanceWithoutThreadActivityPx += stepPx; + } + + roundsWithoutMovement = scroll.moved ? 0 : roundsWithoutMovement + 1; + progress = nextProgress; + + if (scroll.atBottom && stagnantRounds >= 6) { + context.log.debug("Stopping X thread scroll after reaching page bottom with no further thread progress."); + break; + } + + if (roundsWithoutMovement >= 2 && stagnantRounds >= 4) { + context.log.debug("Stopping X thread scroll after repeated downward scrolls no longer move the page."); + break; + } + + if (distanceWithoutThreadActivityPx >= 24_000 && stagnantRounds >= 12) { + context.log.debug("Stopping X thread scroll after a long stretch with no thread-related progress."); + break; + } + } +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread.ts new file mode 100644 index 0000000..9a1caa6 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/thread.ts @@ -0,0 +1,316 @@ +import type { ExtractedDocument } from "../../extract/document"; +import { + formatMediaList, + formatTweetAuthor, + getLegacy, + getTweetAuthorMetadata, + isRecord, + normalizeTitle, + toXTweet, + unwrapTweetResult, +} from "./shared"; +import type { JsonObject, XQuotedTweet, XTweet } from "./types"; + +interface ParsedThreadTweet extends XTweet { + userId?: string; + conversationId?: string; + inReplyToUserId?: string; + sortTimestamp: number; +} + +function compareTweetIds(left: string, right: string): number { + try { + const leftId = BigInt(left); + const rightId = BigInt(right); + if (leftId === rightId) { + return 0; + } + return leftId < rightId ? -1 : 1; + } catch { + return left.localeCompare(right); + } +} + +function toTimestamp(value: string | undefined): number { + if (!value) { + return 0; + } + const parsed = Date.parse(value); + return Number.isNaN(parsed) ? 0 : parsed; +} + +function scoreParsedTweet(tweet: ParsedThreadTweet): number { + return ( + (tweet.text ? 4 : 0) + + (tweet.author ? 2 : 0) + + (tweet.authorName ? 2 : 0) + + (tweet.media.length > 0 ? 1 : 0) + ); +} + +function toParsedThreadTweet(tweet: JsonObject, pageUrl: string): ParsedThreadTweet { + const legacy = getLegacy(tweet); + const xTweet = toXTweet(tweet, pageUrl); + + return { + ...xTweet, + userId: typeof legacy.user_id_str === "string" ? legacy.user_id_str : undefined, + conversationId: typeof legacy.conversation_id_str === "string" ? legacy.conversation_id_str : undefined, + inReplyToUserId: typeof legacy.in_reply_to_user_id_str === "string" ? legacy.in_reply_to_user_id_str : undefined, + sortTimestamp: toTimestamp(xTweet.createdAt), + }; +} + +function collectTweetFromItemContent( + itemContent: unknown, + pageUrl: string, + tweets: Map, +): void { + if (!isRecord(itemContent)) { + return; + } + + const tweet = unwrapTweetResult( + isRecord(itemContent.tweet_results) ? itemContent.tweet_results.result : null, + ); + if (!tweet || typeof tweet.rest_id !== "string") { + return; + } + + const parsed = toParsedThreadTweet(tweet, pageUrl); + const existing = tweets.get(parsed.id); + if (!existing || scoreParsedTweet(parsed) >= scoreParsedTweet(existing)) { + tweets.set(parsed.id, parsed); + } +} + +function collectTweetsFromItems( + items: unknown, + pageUrl: string, + tweets: Map, +): void { + if (!Array.isArray(items)) { + return; + } + + for (const item of items) { + if (!isRecord(item)) { + continue; + } + + if (isRecord(item.item) && isRecord(item.item.itemContent)) { + collectTweetFromItemContent(item.item.itemContent, pageUrl, tweets); + continue; + } + + if (isRecord(item.itemContent)) { + collectTweetFromItemContent(item.itemContent, pageUrl, tweets); + } + } +} + +function getInstructions(payload: unknown): unknown[] { + if (!isRecord(payload) || !isRecord(payload.data)) { + return []; + } + + const { data } = payload; + return ( + (isRecord(data.threaded_conversation_with_injections_v2) && + Array.isArray(data.threaded_conversation_with_injections_v2.instructions) + ? data.threaded_conversation_with_injections_v2.instructions + : undefined) ?? + (isRecord(data.threaded_conversation_with_injections) && + Array.isArray(data.threaded_conversation_with_injections.instructions) + ? data.threaded_conversation_with_injections.instructions + : undefined) ?? + (isRecord(data.tweetResult) && + isRecord(data.tweetResult.result) && + isRecord(data.tweetResult.result.timeline) && + Array.isArray(data.tweetResult.result.timeline.instructions) + ? data.tweetResult.result.timeline.instructions + : []) + ); +} + +function parseTweetDetailPayload(payload: unknown, pageUrl: string): ParsedThreadTweet[] { + const tweets = new Map(); + + const instructions = getInstructions(payload); + for (const instruction of instructions) { + if (!isRecord(instruction)) { + continue; + } + + collectTweetsFromItems(instruction.moduleItems, pageUrl, tweets); + + if (!Array.isArray(instruction.entries)) { + continue; + } + + for (const entry of instruction.entries) { + if (!isRecord(entry)) { + continue; + } + + const content = isRecord(entry.content) ? entry.content : {}; + collectTweetFromItemContent(content.itemContent, pageUrl, tweets); + collectTweetsFromItems(content.items, pageUrl, tweets); + } + } + + return Array.from(tweets.values()); +} + +function buildContinuousThread(tweets: ParsedThreadTweet[], statusId: string): ParsedThreadTweet[] { + const byId = new Map(); + for (const tweet of tweets) { + const existing = byId.get(tweet.id); + if (!existing || scoreParsedTweet(tweet) >= scoreParsedTweet(existing)) { + byId.set(tweet.id, tweet); + } + } + + const rootTweet = byId.get(statusId); + if (!rootTweet?.userId || !rootTweet.conversationId) { + return []; + } + + const candidates = Array.from(byId.values()).filter( + (tweet) => + tweet.id === statusId || + (tweet.userId === rootTweet.userId && tweet.conversationId === rootTweet.conversationId), + ); + + const repliesByParent = new Map(); + for (const tweet of candidates) { + if (!tweet.inReplyTo || tweet.id === statusId) { + continue; + } + const bucket = repliesByParent.get(tweet.inReplyTo) ?? []; + bucket.push(tweet); + bucket.sort((left, right) => { + if (left.sortTimestamp !== right.sortTimestamp) { + return left.sortTimestamp - right.sortTimestamp; + } + return compareTweetIds(left.id, right.id); + }); + repliesByParent.set(tweet.inReplyTo, bucket); + } + + const ancestorPath: ParsedThreadTweet[] = [rootTweet]; + const ancestorSeen = new Set([rootTweet.id]); + let currentAncestor = rootTweet; + + while (currentAncestor.inReplyTo) { + const parent = byId.get(currentAncestor.inReplyTo); + if (!parent || ancestorSeen.has(parent.id)) { + break; + } + ancestorPath.unshift(parent); + ancestorSeen.add(parent.id); + currentAncestor = parent; + } + + const chain = ancestorPath.slice(); + const seen = new Set(chain.map((tweet) => tweet.id)); + let currentId = rootTweet.id; + + while (true) { + const next = (repliesByParent.get(currentId) ?? []).find((tweet) => !seen.has(tweet.id)); + if (!next) { + break; + } + chain.push(next); + seen.add(next.id); + currentId = next.id; + } + + return chain; +} + +export function extractThreadTweetsFromPayloads( + payloads: unknown[], + statusId: string, + pageUrl: string, +): XTweet[] { + const parsedTweets: ParsedThreadTweet[] = []; + + for (const payload of payloads) { + parsedTweets.push(...parseTweetDetailPayload(payload, pageUrl)); + } + + return buildContinuousThread(parsedTweets, statusId).map(({ sortTimestamp: _sortTimestamp, ...tweet }) => tweet); +} + +function buildQuotedTweetMarkdown(quotedTweet: XQuotedTweet): string { + const author = quotedTweet.author ? `@${quotedTweet.author}` : "Unknown"; + const name = quotedTweet.authorName ? `${quotedTweet.authorName} ` : ""; + const lines: string[] = [`Quoted Tweet${quotedTweet.author || quotedTweet.authorName ? `: ${name}${author}`.trim() : ""}`]; + + if (quotedTweet.text) { + lines.push(...quotedTweet.text.split("\n")); + } + + for (const mediaLine of formatMediaList(quotedTweet.media)) { + lines.push(mediaLine); + } + + return lines.map((line) => (line ? `> ${line}` : ">")).join("\n"); +} + +function buildThreadMarkdown(tweets: XTweet[]): string { + return tweets + .map((tweet, index) => { + const lines: string[] = []; + const author = tweet.author ? `@${tweet.author}` : "Unknown"; + const name = tweet.authorName ? `${tweet.authorName} ` : ""; + lines.push(`## ${index + 1}. ${name}${author}`.trim()); + if (tweet.createdAt) { + lines.push(`_Published: ${tweet.createdAt}_`); + } + lines.push(tweet.text || "(No text)"); + const mediaLines = formatMediaList(tweet.media); + if (mediaLines.length > 0) { + lines.push(mediaLines.map((line) => `- ${line}`).join("\n")); + } + if (tweet.quotedTweet) { + lines.push(buildQuotedTweetMarkdown(tweet.quotedTweet)); + } + return lines.join("\n\n"); + }) + .join("\n\n"); +} + +export function extractThreadDocumentFromPayloads( + payloads: unknown[], + statusId: string, + pageUrl: string, +): ExtractedDocument | null { + const tweets = extractThreadTweetsFromPayloads(payloads, statusId, pageUrl); + if (tweets.length <= 1) { + return null; + } + + const rootTweet = tweets[0]; + const rootAuthor = formatTweetAuthor(rootTweet); + + return { + url: pageUrl, + canonicalUrl: rootTweet.url, + title: normalizeTitle(rootTweet.text, "X Thread"), + author: rootAuthor, + siteName: "X", + publishedAt: rootTweet.createdAt, + summary: rootTweet.text.slice(0, 200) || undefined, + adapter: "x", + metadata: { + kind: "x/thread", + tweetId: rootTweet.id, + tweetCount: tweets.length, + lastTweetId: tweets[tweets.length - 1]?.id, + ...getTweetAuthorMetadata(rootTweet), + }, + content: [{ type: "markdown", markdown: buildThreadMarkdown(tweets) }], + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/types.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/types.ts new file mode 100644 index 0000000..509852d --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/x/types.ts @@ -0,0 +1,36 @@ +export type JsonObject = Record; + +export interface XUser { + name?: string; + screenName?: string; +} + +export interface XMedia { + type: string; + url: string; + alt?: string; +} + +export interface XQuotedTweet { + id: string; + author?: string; + authorName?: string; + text: string; + url: string; + media: XMedia[]; +} + +export interface XTweet { + id: string; + author?: string; + authorName?: string; + text: string; + likes: number; + retweets: number; + replies: number; + createdAt?: string; + inReplyTo?: string; + url: string; + media: XMedia[]; + quotedTweet?: XQuotedTweet; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/index.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/index.ts new file mode 100644 index 0000000..4119b9c --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/index.ts @@ -0,0 +1,33 @@ +import type { Adapter } from "../types"; +import { collectMediaFromDocument } from "../../media/markdown-media"; +import { extractYouTubeTranscriptDocument } from "./transcript"; +import { isYouTubeHost, parseYouTubeVideoId } from "./utils"; + +export const youtubeAdapter: Adapter = { + name: "youtube", + match(input) { + return isYouTubeHost(input.url.hostname); + }, + async process(context) { + const videoId = parseYouTubeVideoId(context.input.url); + if (!videoId) { + return { + status: "no_document", + }; + } + + context.log.info(`Loading ${context.input.url.toString()} with youtube adapter`); + const document = await extractYouTubeTranscriptDocument(context, videoId); + if (!document) { + return { + status: "no_document", + }; + } + + return { + status: "ok", + document, + media: collectMediaFromDocument(document), + }; + }, +}; diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/transcript.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/transcript.ts new file mode 100644 index 0000000..f13701a --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/transcript.ts @@ -0,0 +1,392 @@ +import type { ExtractedDocument } from "../../extract/document"; +import { detectInteractionGate } from "../../browser/interaction-gates"; +import { + buildYouTubeThumbnailCandidates, + parseYouTubeDescriptionChapters, + renderYouTubeTranscriptMarkdown, + type YouTubeChapter, + type YouTubeTranscriptSegment, +} from "./utils"; + +interface CaptionInfo { + captionUrl: string; + language: string; + kind: string; + available: string[]; + title?: string; + author?: string; + authorUrl?: string; + channelId?: string; + description?: string; + publishedAt?: string; + viewCount?: number; + durationSeconds?: number; + keywords: string[]; + category?: string; + isLiveContent?: boolean; + coverImages: string[]; +} + +function normalizeUrl(url: string | undefined): string | undefined { + if (!url) { + return undefined; + } + + try { + const parsed = new URL(url); + if (parsed.protocol === "http:") { + parsed.protocol = "https:"; + } + return parsed.toString(); + } catch { + return url; + } +} + +function buildSummary(description: string | undefined, segments: YouTubeTranscriptSegment[]): string | undefined { + const descriptionSummary = description + ?.replace(/\r\n/g, "\n") + .split("\n") + .map((line) => line.trim()) + .find((line) => line && !/^https?:\/\//i.test(line)); + + if (descriptionSummary) { + return descriptionSummary.slice(0, 240); + } + + const transcriptSummary = segments + .slice(0, 8) + .map((segment) => segment.text) + .join(" ") + .slice(0, 240) + .trim(); + + return transcriptSummary || undefined; +} + +async function canFetchThumbnail(url: string): Promise { + try { + const response = await fetch(url, { method: "HEAD", redirect: "follow" }); + if (response.ok) { + return true; + } + + if (response.status === 405) { + const fallbackResponse = await fetch(url, { + method: "GET", + headers: { Range: "bytes=0-0" }, + redirect: "follow", + }); + return fallbackResponse.ok; + } + } catch { + return false; + } + + return false; +} + +async function resolveBestCoverImage(videoId: string, coverImages: string[]): Promise { + const candidates = buildYouTubeThumbnailCandidates(videoId, coverImages); + + for (const candidate of candidates) { + if (await canFetchThumbnail(candidate)) { + return candidate; + } + } + + return candidates[0]; +} + +export async function extractYouTubeTranscriptDocument( + context: Parameters[0], + videoId: string, +): Promise { + const videoUrl = `https://www.youtube.com/watch?v=${videoId}`; + await context.browser.goto(videoUrl, context.timeoutMs); + + const interaction = await detectInteractionGate(context.browser); + if (interaction) { + context.log.debug(`Interaction gate detected on YouTube: ${interaction.provider}`); + return null; + } + + try { + await context.network.waitForIdle({ + idleMs: 1_000, + timeoutMs: Math.min(context.timeoutMs, 8_000), + }); + } catch { + context.log.debug("Network idle timed out on YouTube load."); + } + + const captionInfo = await context.browser.evaluate(` + (async () => { + function readText(value) { + if (!value) return undefined; + if (typeof value === 'string') { + const text = value.trim(); + return text || undefined; + } + if (typeof value.simpleText === 'string') { + const text = value.simpleText.trim(); + return text || undefined; + } + if (Array.isArray(value.runs)) { + const text = value.runs + .map((run) => typeof run?.text === 'string' ? run.text : '') + .join('') + .trim(); + return text || undefined; + } + return undefined; + } + + function parsePositiveInteger(value) { + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) { + return Math.floor(value); + } + if (typeof value !== 'string') { + return undefined; + } + const normalized = value.replace(/[^\\d]/g, ''); + if (!normalized) { + return undefined; + } + const parsed = Number.parseInt(normalized, 10); + return Number.isFinite(parsed) ? parsed : undefined; + } + + const apiKey = window.ytcfg?.data_?.INNERTUBE_API_KEY; + const playerResponse = window.ytInitialPlayerResponse; + const videoDetails = playerResponse?.videoDetails || {}; + const microformat = playerResponse?.microformat?.playerMicroformatRenderer || {}; + const title = + videoDetails.title || + readText(microformat.title) || + document.title.replace(/ - YouTube$/, '').trim(); + const author = + videoDetails.author || + microformat.ownerChannelName || + document.querySelector('link[itemprop="name"]')?.getAttribute('content') || + undefined; + const authorUrl = + microformat.ownerProfileUrl || + (typeof videoDetails.channelId === 'string' && videoDetails.channelId + ? 'https://www.youtube.com/channel/' + videoDetails.channelId + : undefined); + const description = + readText(microformat.description) || + (typeof videoDetails.shortDescription === 'string' ? videoDetails.shortDescription.trim() : undefined); + const keywords = Array.isArray(videoDetails.keywords) + ? videoDetails.keywords.filter((keyword) => typeof keyword === 'string' && keyword.trim()) + : []; + const thumbnails = [ + ...(Array.isArray(videoDetails.thumbnail?.thumbnails) ? videoDetails.thumbnail.thumbnails : []), + ...(Array.isArray(microformat.thumbnail?.thumbnails) ? microformat.thumbnail.thumbnails : []), + ] + .filter((thumbnail) => typeof thumbnail?.url === 'string' && thumbnail.url) + .sort((left, right) => ((right?.width || 0) * (right?.height || 0)) - ((left?.width || 0) * (left?.height || 0))) + .map((thumbnail) => thumbnail.url); + + if (!apiKey) { + return { error: 'INNERTUBE_API_KEY not found on page' }; + } + + const response = await fetch('/youtubei/v1/player?key=' + apiKey + '&prettyPrint=false', { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + context: { client: { clientName: 'ANDROID', clientVersion: '20.10.38' } }, + videoId: ${JSON.stringify(videoId)} + }) + }); + + if (!response.ok) { + return { error: 'InnerTube player API returned HTTP ' + response.status }; + } + + const data = await response.json(); + const renderer = data.captions?.playerCaptionsTracklistRenderer; + if (!renderer?.captionTracks?.length) { + return { error: 'No captions available for this video' }; + } + + const tracks = renderer.captionTracks; + const track = tracks.find((item) => item.kind !== 'asr') || tracks[0]; + + return { + captionUrl: track.baseUrl, + language: track.languageCode, + kind: track.kind || 'manual', + available: tracks.map((item) => { + const languageLabel = readText(item.name) || item.languageCode; + return item.kind === 'asr' + ? languageLabel + ' [' + item.languageCode + ', auto]' + : languageLabel + ' [' + item.languageCode + ']'; + }), + title, + author, + authorUrl, + channelId: typeof videoDetails.channelId === 'string' ? videoDetails.channelId : undefined, + description, + publishedAt: + (typeof microformat.publishDate === 'string' && microformat.publishDate) || + (typeof microformat.uploadDate === 'string' && microformat.uploadDate) || + document.querySelector('meta[itemprop="datePublished"]')?.getAttribute('content') || + undefined, + viewCount: parsePositiveInteger(videoDetails.viewCount) ?? parsePositiveInteger(microformat.viewCount), + durationSeconds: parsePositiveInteger(videoDetails.lengthSeconds), + keywords, + category: typeof microformat.category === 'string' ? microformat.category : undefined, + isLiveContent: Boolean(videoDetails.isLiveContent || microformat.isLiveContent), + coverImages: thumbnails, + }; + })() + `); + + if ("error" in captionInfo) { + context.log.debug(`YouTube transcript unavailable: ${captionInfo.error}`); + return null; + } + + const segments = await context.browser.evaluate(` + (async () => { + const response = await fetch(${JSON.stringify(captionInfo.captionUrl)}); + const xml = await response.text(); + if (!xml) { + return { error: 'Caption XML is empty' }; + } + + function getAttr(tag, name) { + const needle = name + '="'; + const index = tag.indexOf(needle); + if (index === -1) return ''; + const valueStart = index + needle.length; + const valueEnd = tag.indexOf('"', valueStart); + if (valueEnd === -1) return ''; + return tag.substring(valueStart, valueEnd); + } + + function decodeEntities(value) { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll(''', "'"); + } + + const marker = xml.includes('

(` + (() => { + const data = window.ytInitialData; + const markers = data?.playerOverlays?.playerOverlayRenderer + ?.decoratedPlayerBarRenderer?.decoratedPlayerBarRenderer + ?.playerBar?.multiMarkersPlayerBarRenderer?.markersMap || []; + const results = []; + + for (const marker of markers) { + const chapters = marker?.value?.chapters; + if (!Array.isArray(chapters)) continue; + for (const chapter of chapters) { + const renderer = chapter?.chapterRenderer; + const title = renderer?.title?.simpleText; + const timeRangeStartMillis = renderer?.timeRangeStartMillis; + if (title && typeof timeRangeStartMillis === 'number') { + results.push({ title, time: Math.floor(timeRangeStartMillis / 1000) }); + } + } + } + + return results; + })() + `).catch(() => []); + + const descriptionChapters = parseYouTubeDescriptionChapters(captionInfo.description); + const chapters = extractedChapters.length > 0 ? extractedChapters : descriptionChapters; + const markdown = renderYouTubeTranscriptMarkdown({ + description: captionInfo.description, + segments, + chapters, + }); + + if (!markdown) { + return null; + } + + const pageUrl = await context.browser.getURL(); + const coverImage = await resolveBestCoverImage(videoId, captionInfo.coverImages); + const summary = buildSummary(captionInfo.description, segments); + + return { + url: pageUrl, + canonicalUrl: pageUrl, + title: captionInfo.title || "YouTube Transcript", + author: captionInfo.author, + publishedAt: captionInfo.publishedAt, + siteName: "YouTube", + summary, + adapter: "youtube", + metadata: { + kind: "youtube/transcript", + videoId, + authorUrl: normalizeUrl(captionInfo.authorUrl), + channelId: captionInfo.channelId, + coverImage, + description: captionInfo.description, + durationSeconds: captionInfo.durationSeconds, + language: captionInfo.language, + captionKind: captionInfo.kind, + availableLanguages: captionInfo.available, + viewCount: captionInfo.viewCount, + keywords: captionInfo.keywords, + category: captionInfo.category, + isLiveContent: captionInfo.isLiveContent, + chapterCount: chapters.length, + }, + content: [{ type: "markdown", markdown }], + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts new file mode 100644 index 0000000..3770df9 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts @@ -0,0 +1,253 @@ +export interface YouTubeTranscriptSegment { + start: number; + end: number; + text: string; +} + +export interface YouTubeChapter { + title: string; + time: number; +} + +interface RenderYouTubeTranscriptMarkdownInput { + description?: string; + segments: YouTubeTranscriptSegment[]; + chapters: YouTubeChapter[]; +} + +const DESCRIPTION_CHAPTER_RE = /^((?:\d{1,2}:)?\d{1,2}:\d{2})(?:\s+[-|:]\s+|\s+)(.+)$/; +const YOUTUBE_THUMBNAIL_VARIANTS = [ + "maxresdefault.jpg", + "sddefault.jpg", + "hqdefault.jpg", + "mqdefault.jpg", + "default.jpg", +]; + +export function isYouTubeHost(hostname: string): boolean { + return [ + "youtube.com", + "www.youtube.com", + "m.youtube.com", + "youtu.be", + ].includes(hostname); +} + +export function parseYouTubeVideoId(url: URL): string | null { + if (url.hostname === "youtu.be") { + return url.pathname.split("/").filter(Boolean)[0] ?? null; + } + + if (url.pathname === "/watch") { + return url.searchParams.get("v"); + } + + const shortsMatch = url.pathname.match(/^\/shorts\/([^/?#]+)/); + if (shortsMatch) { + return shortsMatch[1]; + } + + const liveMatch = url.pathname.match(/^\/live\/([^/?#]+)/); + if (liveMatch) { + return liveMatch[1]; + } + + return null; +} + +function parseTimestampValue(raw: string): number | null { + const parts = raw + .split(":") + .map((part) => Number.parseInt(part, 10)) + .filter((part) => Number.isFinite(part)); + + if (parts.length < 2 || parts.length > 3) { + return null; + } + + if (parts.some((part) => part < 0)) { + return null; + } + + if (parts.length === 2) { + const [minutes, seconds] = parts; + return minutes * 60 + seconds; + } + + const [hours, minutes, seconds] = parts; + return hours * 3600 + minutes * 60 + seconds; +} + +export function formatTimestamp(totalSeconds: number): string { + const rounded = Math.max(0, Math.floor(totalSeconds)); + const hours = Math.floor(rounded / 3600); + const minutes = Math.floor((rounded % 3600) / 60); + const seconds = rounded % 60; + + if (hours > 0) { + return `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`; + } + return `${minutes}:${String(seconds).padStart(2, "0")}`; +} + +export function formatTimestampRange(start: number, end: number): string { + const safeStart = Math.max(0, start); + const safeEnd = Math.max(safeStart, end); + return `[${formatTimestamp(safeStart)} -> ${formatTimestamp(safeEnd)}]`; +} + +export function normalizeYouTubeChapters(chapters: YouTubeChapter[]): YouTubeChapter[] { + const seenTimes = new Set(); + + return chapters + .map((chapter) => ({ + title: chapter.title.trim(), + time: Math.max(0, Math.floor(chapter.time)), + })) + .filter((chapter) => chapter.title) + .sort((left, right) => left.time - right.time) + .filter((chapter) => { + if (seenTimes.has(chapter.time)) { + return false; + } + seenTimes.add(chapter.time); + return true; + }); +} + +export function parseYouTubeDescriptionChapters(description?: string | null): YouTubeChapter[] { + if (!description) { + return []; + } + + const chapters: YouTubeChapter[] = []; + const seen = new Set(); + + for (const rawLine of description.replace(/\r\n/g, "\n").split("\n")) { + const line = rawLine.trim(); + if (!line) { + continue; + } + + const match = line.match(DESCRIPTION_CHAPTER_RE); + if (!match) { + continue; + } + + const time = parseTimestampValue(match[1]); + const title = match[2]?.trim(); + if (time === null || !title) { + continue; + } + + const key = `${time}:${title.toLowerCase()}`; + if (seen.has(key)) { + continue; + } + + seen.add(key); + chapters.push({ title, time }); + } + + const normalized = normalizeYouTubeChapters(chapters); + if (normalized.length >= 2) { + return normalized; + } + + if (normalized.length === 1 && normalized[0]?.time === 0) { + return normalized; + } + + return []; +} + +function renderDescriptionMarkdown(description: string): string { + return description + .replace(/\r\n/g, "\n") + .trim() + .split(/\n{2,}/) + .map((block) => block.split("\n").map((line) => line.trimEnd()).join(" \n")) + .join("\n\n") + .trim(); +} + +function renderSegmentLine(segment: YouTubeTranscriptSegment): string { + return `${formatTimestampRange(segment.start, segment.end)} ${segment.text}`; +} + +export function renderYouTubeTranscriptMarkdown({ + description, + segments, + chapters, +}: RenderYouTubeTranscriptMarkdownInput): string { + if (segments.length === 0) { + return ""; + } + + const parts: string[] = []; + const normalizedDescription = description?.trim(); + const transcriptEnd = segments.reduce((maxEnd, segment) => Math.max(maxEnd, segment.end, segment.start), 0); + const normalizedChapters = normalizeYouTubeChapters(chapters).filter( + (chapter) => transcriptEnd <= 0 || chapter.time < transcriptEnd, + ); + + if (normalizedDescription) { + parts.push("## Description"); + parts.push(renderDescriptionMarkdown(normalizedDescription)); + } + + if (normalizedChapters.length > 0) { + parts.push("## Chapters"); + + for (let index = 0; index < normalizedChapters.length; index += 1) { + const chapter = normalizedChapters[index]; + const nextChapter = normalizedChapters[index + 1]; + const chapterEnd = nextChapter ? nextChapter.time : transcriptEnd; + const chapterSegments = segments.filter( + (segment) => segment.start >= chapter.time && segment.start < chapterEnd, + ); + + parts.push(`### ${chapter.title} ${formatTimestampRange(chapter.time, chapterEnd)}`); + + if (chapterSegments.length > 0) { + parts.push(chapterSegments.map(renderSegmentLine).join("\n")); + } + } + } else { + parts.push("## Transcript"); + parts.push(segments.map(renderSegmentLine).join("\n")); + } + + return parts.filter(Boolean).join("\n\n").trim(); +} + +function normalizeThumbnailKey(url: string): string { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return url; + } +} + +export function buildYouTubeThumbnailCandidates(videoId: string, listedUrls: string[]): string[] { + const candidates = [ + ...YOUTUBE_THUMBNAIL_VARIANTS.map((variant) => `https://i.ytimg.com/vi/${videoId}/${variant}`), + ...listedUrls, + ]; + + const seen = new Set(); + return candidates.filter((candidate) => { + if (!candidate) { + return false; + } + + const key = normalizeThumbnailKey(candidate); + if (seen.has(key)) { + return false; + } + + seen.add(key); + return true; + }); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts new file mode 100644 index 0000000..8c7eaef --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts @@ -0,0 +1,258 @@ +import { EventEmitter } from "node:events"; +import WebSocket from "ws"; + +type JsonObject = Record; + +interface CdpPendingCommand { + resolve(value: unknown): void; + reject(error: unknown): void; + method: string; +} + +interface CdpErrorShape { + message?: string; +} + +interface CdpCommandResult { + result?: T; + error?: CdpErrorShape; +} + +interface CreatePageSessionOptions { + initialUrl?: string; + visible?: boolean; +} + +export class TargetSession extends EventEmitter { + constructor( + private readonly client: CdpClient, + public readonly targetId: string, + public readonly sessionId: string, + ) { + super(); + } + + async send(method: string, params: JsonObject = {}): Promise { + return this.client.sendSessionCommand(this.sessionId, method, params); + } + + handleEvent(method: string, params: JsonObject): void { + this.emit(method, params); + this.emit("event", { method, params }); + } + + async waitForEvent( + method: string, + predicate?: (params: T) => boolean, + timeoutMs = 30_000, + ): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.off(method, listener); + reject(new Error(`Timed out waiting for ${method}`)); + }, timeoutMs); + + const listener = (params: T): void => { + if (predicate && !predicate(params)) { + return; + } + clearTimeout(timeout); + this.off(method, listener); + resolve(params); + }; + + this.on(method, listener); + }); + } +} + +export class CdpClient { + private readonly ws: WebSocket; + private readonly pending = new Map(); + private readonly sessions = new Map(); + private nextId = 1; + + private constructor(ws: WebSocket) { + this.ws = ws; + this.ws.on("message", (raw) => { + this.handleMessage(raw.toString()); + }); + } + + static async connect(browserWsUrl: string): Promise { + const ws = await new Promise((resolve, reject) => { + const socket = new WebSocket(browserWsUrl); + socket.once("open", () => resolve(socket)); + socket.once("error", (error) => reject(error)); + }); + + return new CdpClient(ws); + } + + private handleMessage(rawMessage: string): void { + const message = JSON.parse(rawMessage) as { + id?: number; + sessionId?: string; + method?: string; + params?: JsonObject; + result?: unknown; + error?: CdpErrorShape; + }; + + if (typeof message.id === "number") { + const pending = this.pending.get(message.id); + if (!pending) { + return; + } + this.pending.delete(message.id); + if (message.error) { + pending.reject(new Error(`${pending.method}: ${message.error.message ?? "Unknown CDP error"}`)); + return; + } + pending.resolve(message.result); + return; + } + + if (typeof message.sessionId === "string" && typeof message.method === "string") { + const session = this.sessions.get(message.sessionId); + if (session) { + session.handleEvent(message.method, (message.params ?? {}) as JsonObject); + } + } + } + + private async sendCommand( + method: string, + params: JsonObject = {}, + sessionId?: string, + ): Promise { + const id = this.nextId; + this.nextId += 1; + + const payload = sessionId ? { id, method, params, sessionId } : { id, method, params }; + + const result = new Promise((resolve, reject) => { + this.pending.set(id, { + resolve: (value) => resolve(value as T), + reject, + method, + }); + }); + + this.ws.send(JSON.stringify(payload)); + return result; + } + + async sendBrowserCommand(method: string, params: JsonObject = {}): Promise { + return this.sendCommand(method, params); + } + + async sendSessionCommand(sessionId: string, method: string, params: JsonObject = {}): Promise { + return this.sendCommand(method, params, sessionId); + } + + private async createPageTarget(initialUrl: string, visible = false): Promise<{ targetId: string }> { + const attempts: JsonObject[] = visible + ? [ + { + url: initialUrl, + newWindow: true, + focus: true, + }, + { + url: initialUrl, + focus: true, + }, + { + url: initialUrl, + }, + ] + : [ + { + url: initialUrl, + hidden: true, + }, + { + url: initialUrl, + background: true, + focus: false, + }, + { + url: initialUrl, + }, + ]; + + let lastError: unknown; + + for (const params of attempts) { + try { + return await this.sendBrowserCommand<{ targetId: string }>("Target.createTarget", params); + } catch (error) { + lastError = error; + } + } + + throw lastError instanceof Error ? lastError : new Error("Target.createTarget failed"); + } + + async createPageSession(options: CreatePageSessionOptions = {}): Promise { + const initialUrl = options.initialUrl ?? "about:blank"; + const created = await this.createPageTarget(initialUrl, Boolean(options.visible)); + const attached = await this.sendBrowserCommand<{ sessionId: string }>("Target.attachToTarget", { + targetId: created.targetId, + flatten: true, + }); + + const session = new TargetSession(this, created.targetId, attached.sessionId); + this.sessions.set(attached.sessionId, session); + + if (options.visible) { + await this.sendBrowserCommand("Target.activateTarget", { + targetId: created.targetId, + }).catch(() => {}); + } + + await session.send("Page.enable"); + await session.send("Runtime.enable"); + await session.send("DOM.enable"); + + if (options.visible) { + await session.send("Page.bringToFront").catch(() => {}); + } + + return session; + } + + async closeTarget(targetId: string): Promise { + try { + await this.sendBrowserCommand("Target.closeTarget", { targetId }); + } catch { + // Target may already be gone. + } + } + + async close(): Promise { + await new Promise((resolve) => { + if (this.ws.readyState === WebSocket.CLOSED) { + resolve(); + return; + } + this.ws.once("close", () => resolve()); + this.ws.close(); + }); + } +} + +export async function evaluateRuntime(session: TargetSession, expression: string): Promise { + const response = await session.send>("Runtime.evaluate", { + expression, + awaitPromise: true, + returnByValue: true, + }); + + if (response.error) { + throw new Error(response.error.message ?? "Runtime.evaluate failed"); + } + + return (response.result?.value as T | undefined) ?? (undefined as T); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts new file mode 100644 index 0000000..1efe4c5 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts @@ -0,0 +1,187 @@ +import { launch, type LaunchedChrome } from "chrome-launcher"; +import WebSocket from "ws"; +import type { Logger } from "../utils/logger"; +import { + cleanChromeLockArtifacts, + ensureChromeProfileDir, + findChromeProcessUsingProfile, + findExistingChromeDebugPort, + hasChromeLockArtifacts, + listChromeProfileEntries, + resolveChromeProfileDir, + shouldRetryChromeLaunchRecovery, +} from "./profile"; + +interface ChromeVersionResponse { + webSocketDebuggerUrl: string; +} + +export interface ChromeConnectOptions { + cdpUrl?: string; + browserPath?: string; + headless?: boolean; + logger?: Logger; + profileDir?: string; +} + +export interface ChromeConnection { + browserWsUrl: string; + origin?: string; + port?: number; + profileDir?: string; + launched: boolean; + close(): Promise; +} + +async function fetchJson(url: string): Promise { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`); + } + return (await response.json()) as T; +} + +async function connectToHttpEndpoint(origin: string): Promise { + const normalizedOrigin = origin.replace(/\/$/, ""); + const version = await fetchJson(`${normalizedOrigin}/json/version`); + return { + browserWsUrl: version.webSocketDebuggerUrl, + origin: normalizedOrigin, + port: Number(new URL(normalizedOrigin).port || 80), + launched: false, + async close() { + // Reused external Chrome, nothing to close here. + }, + }; +} + +async function tryReuseChrome(profileDir: string, logger?: Logger): Promise { + const port = await findExistingChromeDebugPort({ profileDir }); + if (!port) { + return null; + } + + const origin = `http://127.0.0.1:${port}`; + try { + const connection = await connectToHttpEndpoint(origin); + logger?.info(`Reusing Chrome debugger at ${origin} for profile ${profileDir}`); + return { + ...connection, + profileDir, + }; + } catch { + // Debugger disappeared between detection and connect. + } + return null; +} + +async function launchFreshChrome( + profileDir: string, + options: Pick, +): Promise { + let launchedChrome: LaunchedChrome | null = null; + try { + launchedChrome = await launch({ + chromePath: options.browserPath, + userDataDir: profileDir, + chromeFlags: [ + "--disable-background-networking", + "--disable-default-apps", + "--disable-popup-blocking", + "--disable-sync", + "--no-first-run", + "--no-default-browser-check", + "--remote-allow-origins=*", + ...(!options.headless ? ["--no-startup-window"] : []), + ...(options.headless ? ["--headless=new"] : []), + ], + }); + + const origin = `http://127.0.0.1:${launchedChrome.port}`; + const version = await fetchJson(`${origin}/json/version`); + + const chrome = launchedChrome; + return { + browserWsUrl: version.webSocketDebuggerUrl, + origin, + port: launchedChrome.port, + profileDir, + launched: true, + async close() { + if (!chrome) return; + await gracefulCloseChrome(chrome, origin); + }, + }; + } catch (error) { + launchedChrome?.kill(); + throw error; + } +} + +async function gracefulCloseChrome(chrome: LaunchedChrome, origin: string): Promise { + try { + const resp = await fetch(`${origin}/json/version`); + const { webSocketDebuggerUrl } = (await resp.json()) as ChromeVersionResponse; + if (webSocketDebuggerUrl) { + const ws = await new Promise((resolve, reject) => { + const socket = new WebSocket(webSocketDebuggerUrl); + socket.once("open", () => resolve(socket)); + socket.once("error", reject); + }); + const id = 1; + ws.send(JSON.stringify({ id, method: "Browser.close" })); + await new Promise((resolve) => { + const timer = setTimeout(() => { ws.close(); resolve(); }, 5_000); + ws.once("close", () => { clearTimeout(timer); resolve(); }); + }); + const exited = await new Promise((resolve) => { + if (chrome.pid && !isProcessAlive(chrome.pid)) { resolve(true); return; } + const timer = setTimeout(() => resolve(false), 3_000); + chrome.process.once("exit", () => { clearTimeout(timer); resolve(true); }); + }); + if (exited) return; + } + } catch {} + chrome.kill(); +} + +function isProcessAlive(pid: number): boolean { + try { process.kill(pid, 0); return true; } catch { return false; } +} + +export async function connectChrome(options: ChromeConnectOptions): Promise { + if (options.cdpUrl) { + if (options.cdpUrl.startsWith("ws://") || options.cdpUrl.startsWith("wss://")) { + return { + browserWsUrl: options.cdpUrl, + launched: false, + async close() {}, + }; + } + return connectToHttpEndpoint(options.cdpUrl); + } + + const profileDir = ensureChromeProfileDir(resolveChromeProfileDir(options.profileDir)); + const reused = await tryReuseChrome(profileDir, options.logger); + if (reused) { + return reused; + } + + options.logger?.warn(`No running Chrome debugger found for profile ${profileDir}. Launching Chrome with that profile.`); + try { + return await launchFreshChrome(profileDir, options); + } catch (error) { + const entries = await listChromeProfileEntries(profileDir); + const shouldRetry = shouldRetryChromeLaunchRecovery({ + hasLockArtifacts: hasChromeLockArtifacts(entries), + hasLiveOwner: findChromeProcessUsingProfile(profileDir), + }); + if (!shouldRetry) { + throw error; + } + + options.logger?.warn(`Chrome launch failed with stale profile locks. Cleaning ${profileDir} and retrying once.`); + cleanChromeLockArtifacts(profileDir); + return await launchFreshChrome(profileDir, options); + } +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts new file mode 100644 index 0000000..4b80302 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts @@ -0,0 +1,100 @@ +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { resolveChromeProfileDir } from "./profile"; +import type { TargetSession } from "./cdp-client"; + +export interface CdpCookie { + name: string; + value: string; + domain: string; + path: string; + expires: number; + size: number; + httpOnly: boolean; + secure: boolean; + session: boolean; + sameSite?: string; + priority?: string; + sameParty?: boolean; + sourceScheme?: string; + sourcePort?: number; + partitionKey?: string; +} + +interface SidecarData { + savedAt: string; + cookies: CdpCookie[]; +} + +export interface CookieSidecarConfig { + urls: readonly string[]; + filename: string; + requiredCookieNames: readonly string[]; + filterCookie?: (cookie: CdpCookie) => boolean; +} + +function sidecarPath(filename: string, profileDir?: string): string { + return join(resolveChromeProfileDir(profileDir), filename); +} + +function hasRequired(cookies: CdpCookie[], names: readonly string[]): boolean { + return names.every((name) => + cookies.some((c) => c.name === name && Boolean(c.value)), + ); +} + +async function getCookies(session: TargetSession, urls: readonly string[]): Promise { + const { cookies } = await session.send<{ cookies: CdpCookie[] }>( + "Network.getCookies", + { urls: [...urls] }, + ); + return cookies ?? []; +} + +export async function exportCookies( + session: TargetSession, + config: CookieSidecarConfig, + profileDir?: string, +): Promise { + const all = await getCookies(session, config.urls); + const filtered = config.filterCookie ? all.filter(config.filterCookie) : all; + if (!hasRequired(filtered, config.requiredCookieNames)) return false; + + const filePath = sidecarPath(config.filename, profileDir); + await mkdir(dirname(filePath), { recursive: true }); + const data: SidecarData = { savedAt: new Date().toISOString(), cookies: filtered }; + await writeFile(filePath, JSON.stringify(data, null, 2)); + return true; +} + +export async function restoreCookies( + session: TargetSession, + config: CookieSidecarConfig, + profileDir?: string, +): Promise { + const live = await getCookies(session, config.urls); + if (hasRequired(live, config.requiredCookieNames)) return false; + + const filePath = sidecarPath(config.filename, profileDir); + const raw = await readFile(filePath, "utf8"); + const data = JSON.parse(raw) as SidecarData; + if (!data.cookies?.length) return false; + + const now = Date.now() / 1000; + const valid = data.cookies.filter((c) => c.session || !c.expires || c.expires > now); + if (!hasRequired(valid, config.requiredCookieNames)) return false; + + await session.send("Network.setCookies", { + cookies: valid.map((c) => ({ + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + httpOnly: c.httpOnly, + secure: c.secure, + sameSite: c.sameSite, + expires: c.expires, + })), + }); + return true; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts new file mode 100644 index 0000000..123e2d9 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts @@ -0,0 +1,123 @@ +import type { WaitForInteractionRequest } from "../adapters/types"; +import type { BrowserSession } from "./session"; + +interface GateSnapshot { + title: string; + currentUrl: string; + bodyText: string; + hasCloudflareTurnstile: boolean; + hasCloudflareChallenge: boolean; + hasRecaptcha: boolean; + hasRecaptchaIframe: boolean; + hasHcaptcha: boolean; + hasHcaptchaIframe: boolean; +} + +export function detectInteractionGateFromSnapshot(snapshot: GateSnapshot): WaitForInteractionRequest | null { + const text = snapshot.bodyText.toLowerCase(); + const title = snapshot.title.toLowerCase(); + const url = snapshot.currentUrl.toLowerCase(); + + if ( + snapshot.hasCloudflareTurnstile || + snapshot.hasCloudflareChallenge || + title.includes("just a moment") || + text.includes("verify you are human") || + text.includes("checking your browser before accessing") || + text.includes("enable javascript and cookies to continue") || + url.includes("/cdn-cgi/challenge-platform/") + ) { + return { + type: "wait_for_interaction", + kind: "cloudflare", + provider: "cloudflare", + reason: "Cloudflare human verification detected", + prompt: "Please complete the Cloudflare verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.", + requiresVisibleBrowser: true, + }; + } + + if ( + snapshot.hasRecaptcha || + snapshot.hasRecaptchaIframe || + text.includes("i'm not a robot") || + text.includes("recaptcha") + ) { + return { + type: "wait_for_interaction", + kind: "recaptcha", + provider: "google_recaptcha", + reason: "Google reCAPTCHA detected", + prompt: "Please complete the reCAPTCHA verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.", + requiresVisibleBrowser: true, + }; + } + + if ( + snapshot.hasHcaptcha || + snapshot.hasHcaptchaIframe || + text.includes("hcaptcha") + ) { + return { + type: "wait_for_interaction", + kind: "hcaptcha", + provider: "hcaptcha", + reason: "hCaptcha verification detected", + prompt: "Please complete the hCaptcha verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.", + requiresVisibleBrowser: true, + }; + } + + return null; +} + +export async function detectInteractionGate(browser: BrowserSession): Promise { + const snapshot = await browser.evaluate(` + (() => { + const bodyText = (document.body?.innerText ?? "").slice(0, 4000); + return { + title: document.title ?? "", + currentUrl: window.location.href, + bodyText, + hasCloudflareTurnstile: Boolean( + document.querySelector( + '.cf-turnstile, [name="cf-turnstile-response"], iframe[src*="challenges.cloudflare.com"]' + ) + ), + hasCloudflareChallenge: Boolean( + document.querySelector( + '#challenge-running, #cf-challenge-running, .challenge-platform, [data-ray], [data-translate="checking_browser"]' + ) + ), + hasRecaptcha: Boolean( + document.querySelector( + '.g-recaptcha, textarea[name="g-recaptcha-response"], iframe[title*="reCAPTCHA"]' + ) + ), + hasRecaptchaIframe: Boolean( + document.querySelector('iframe[src*="google.com/recaptcha"], iframe[src*="recaptcha/api2"]') + ), + hasHcaptcha: Boolean( + document.querySelector( + '.h-captcha, textarea[name="h-captcha-response"], iframe[title*="hCaptcha"]' + ) + ), + hasHcaptchaIframe: Boolean( + document.querySelector('iframe[src*="hcaptcha.com"]') + ), + }; + })() + `).catch(() => ({ + title: "", + currentUrl: "", + bodyText: "", + hasCloudflareTurnstile: false, + hasCloudflareChallenge: false, + hasRecaptcha: false, + hasRecaptchaIframe: false, + hasHcaptcha: false, + hasHcaptchaIframe: false, + })); + + return detectInteractionGateFromSnapshot(snapshot); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts new file mode 100644 index 0000000..8107910 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts @@ -0,0 +1,235 @@ +import type { TargetSession } from "./cdp-client"; +import type { Logger } from "../utils/logger"; + +type JsonObject = Record; + +export interface NetworkEntry { + requestId: string; + url: string; + method: string; + resourceType: string; + timestamp: number; + requestHeaders?: Record; + requestBody?: string; + status?: number; + statusText?: string; + responseHeaders?: Record; + mimeType?: string; + body?: string; + bodyBase64?: boolean; + bodyError?: string; + failed?: boolean; + failureReason?: string; + finished: boolean; +} + +function normalizeHeaders(headers: unknown): Record | undefined { + if (!headers || typeof headers !== "object") { + return undefined; + } + return Object.fromEntries( + Object.entries(headers as Record).map(([key, value]) => [key, String(value)]), + ); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export class NetworkJournal { + private readonly entries = new Map(); + private lastActivityAt = Date.now(); + private started = false; + + constructor( + private readonly session: TargetSession, + private readonly log: Logger, + ) {} + + async start(): Promise { + if (this.started) { + return; + } + + this.started = true; + this.session.on("Network.requestWillBeSent", this.handleRequestWillBeSent); + this.session.on("Network.responseReceived", this.handleResponseReceived); + this.session.on("Network.loadingFinished", this.handleLoadingFinished); + this.session.on("Network.loadingFailed", this.handleLoadingFailed); + await this.session.send("Network.enable"); + } + + stop(): void { + if (!this.started) { + return; + } + this.session.off("Network.requestWillBeSent", this.handleRequestWillBeSent); + this.session.off("Network.responseReceived", this.handleResponseReceived); + this.session.off("Network.loadingFinished", this.handleLoadingFinished); + this.session.off("Network.loadingFailed", this.handleLoadingFailed); + this.started = false; + } + + private touch(): void { + this.lastActivityAt = Date.now(); + } + + private readonly handleRequestWillBeSent = (params: JsonObject): void => { + const requestId = typeof params.requestId === "string" ? params.requestId : undefined; + const request = params.request as JsonObject | undefined; + if (!requestId || !request) { + return; + } + + this.touch(); + this.entries.set(requestId, { + requestId, + url: String(request.url ?? ""), + method: String(request.method ?? "GET"), + resourceType: String(params.type ?? "Other"), + timestamp: Date.now(), + requestHeaders: normalizeHeaders(request.headers), + requestBody: typeof request.postData === "string" ? request.postData : undefined, + finished: false, + }); + }; + + private readonly handleResponseReceived = (params: JsonObject): void => { + const requestId = typeof params.requestId === "string" ? params.requestId : undefined; + const response = params.response as JsonObject | undefined; + if (!requestId || !response) { + return; + } + + this.touch(); + const existing = this.entries.get(requestId); + if (!existing) { + return; + } + + existing.status = typeof response.status === "number" ? response.status : undefined; + existing.statusText = typeof response.statusText === "string" ? response.statusText : undefined; + existing.responseHeaders = normalizeHeaders(response.headers); + existing.mimeType = typeof response.mimeType === "string" ? response.mimeType : undefined; + this.entries.set(requestId, existing); + }; + + private readonly handleLoadingFinished = (params: JsonObject): void => { + const requestId = typeof params.requestId === "string" ? params.requestId : undefined; + if (!requestId) { + return; + } + + this.touch(); + const existing = this.entries.get(requestId); + if (!existing) { + return; + } + existing.finished = true; + this.entries.set(requestId, existing); + }; + + private readonly handleLoadingFailed = (params: JsonObject): void => { + const requestId = typeof params.requestId === "string" ? params.requestId : undefined; + if (!requestId) { + return; + } + + this.touch(); + const existing = this.entries.get(requestId); + if (!existing) { + return; + } + existing.finished = true; + existing.failed = true; + existing.failureReason = typeof params.errorText === "string" ? params.errorText : "Unknown error"; + this.entries.set(requestId, existing); + }; + + getEntries(): NetworkEntry[] { + return Array.from(this.entries.values()); + } + + findEntries(predicate: (entry: NetworkEntry) => boolean): NetworkEntry[] { + return this.getEntries().filter(predicate); + } + + async waitForIdle(options: { idleMs?: number; timeoutMs?: number } = {}): Promise { + const idleMs = options.idleMs ?? 1_200; + const timeoutMs = options.timeoutMs ?? 15_000; + const startedAt = Date.now(); + + while (Date.now() - startedAt < timeoutMs) { + if (Date.now() - this.lastActivityAt >= idleMs) { + return; + } + await sleep(Math.min(150, idleMs)); + } + + throw new Error("Timed out waiting for network idle"); + } + + async waitForResponse( + predicate: (entry: NetworkEntry) => boolean, + options: { timeoutMs?: number } = {}, + ): Promise { + const timeoutMs = options.timeoutMs ?? 10_000; + const startedAt = Date.now(); + + while (Date.now() - startedAt < timeoutMs) { + const matched = this.getEntries().find((entry) => entry.finished && predicate(entry)); + if (matched) { + return matched; + } + await sleep(150); + } + + throw new Error("Timed out waiting for matching network response"); + } + + async ensureBody(entry: NetworkEntry): Promise { + if (entry.body !== undefined) { + return entry.body; + } + if (entry.bodyError || entry.failed || !entry.finished) { + return undefined; + } + + try { + const result = await this.session.send<{ body: string; base64Encoded: boolean }>("Network.getResponseBody", { + requestId: entry.requestId, + }); + entry.bodyBase64 = result.base64Encoded; + entry.body = result.base64Encoded ? Buffer.from(result.body, "base64").toString("utf8") : result.body; + return entry.body; + } catch (error) { + entry.bodyError = error instanceof Error ? error.message : String(error); + this.log.debug(`Failed to fetch response body for ${entry.url}: ${entry.bodyError}`); + return undefined; + } + } + + async getJsonBody(entry: NetworkEntry): Promise { + const body = await this.ensureBody(entry); + if (!body) { + return null; + } + + try { + return JSON.parse(body); + } catch { + return null; + } + } + + async toJSON(options: { includeBodies?: boolean } = {}): Promise { + const entries = this.getEntries(); + if (!options.includeBodies) { + return entries; + } + + await Promise.all(entries.map((entry) => this.ensureBody(entry))); + return entries; + } +} + diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts new file mode 100644 index 0000000..4b55958 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts @@ -0,0 +1,105 @@ +import type { BrowserSession } from "./session"; + +export interface CapturedPageSnapshot { + html: string; + finalUrl: string; +} + +export const CAPTURE_NORMALIZED_PAGE_SCRIPT = String.raw` +(() => { + const baseUrl = document.baseURI || location.href; + const htmlClone = document.documentElement.cloneNode(true); + + function materializeShadowDom(sourceRoot, cloneRoot) { + const sourceElements = Array.from(sourceRoot.querySelectorAll("*")); + const cloneElements = Array.from(cloneRoot.querySelectorAll("*")); + + for (let index = sourceElements.length - 1; index >= 0; index -= 1) { + const sourceElement = sourceElements[index]; + const cloneElement = cloneElements[index]; + const shadowRoot = sourceElement && sourceElement.shadowRoot; + if (!shadowRoot || !cloneElement || !shadowRoot.innerHTML) { + continue; + } + + if (cloneElement.tagName && cloneElement.tagName.includes("-")) { + const wrapper = document.createElement("div"); + wrapper.setAttribute("data-shadow-host", cloneElement.tagName.toLowerCase()); + wrapper.innerHTML = shadowRoot.innerHTML; + cloneElement.replaceWith(wrapper); + } else { + cloneElement.innerHTML = shadowRoot.innerHTML; + } + } + } + + function toAbsolute(url) { + if (!url) return url; + try { + return new URL(url, baseUrl).href; + } catch { + return url; + } + } + + function absolutizeAttribute(root, selector, attribute) { + root.querySelectorAll(selector).forEach((element) => { + const value = element.getAttribute(attribute); + if (!value) return; + const absolute = toAbsolute(value); + if (absolute) { + element.setAttribute(attribute, absolute); + } + }); + } + + function absolutizeSrcset(root, selector) { + root.querySelectorAll(selector).forEach((element) => { + const srcset = element.getAttribute("srcset"); + if (!srcset) return; + element.setAttribute( + "srcset", + srcset + .split(",") + .map((part) => { + const trimmed = part.trim(); + if (!trimmed) return ""; + const [url, ...descriptor] = trimmed.split(/\s+/); + const absolute = toAbsolute(url); + return descriptor.length > 0 ? absolute + " " + descriptor.join(" ") : absolute; + }) + .filter(Boolean) + .join(", "), + ); + }); + } + + materializeShadowDom(document.documentElement, htmlClone); + + htmlClone + .querySelectorAll("img[data-src], video[data-src], audio[data-src], source[data-src]") + .forEach((element) => { + const dataSource = element.getAttribute("data-src"); + const current = element.getAttribute("src"); + if (dataSource && (!current || current === "" || current.startsWith("data:"))) { + element.setAttribute("src", dataSource); + } + }); + + absolutizeAttribute(htmlClone, "a[href]", "href"); + absolutizeAttribute(htmlClone, "img[src], video[src], audio[src], source[src], iframe[src]", "src"); + absolutizeAttribute(htmlClone, "video[poster]", "poster"); + absolutizeSrcset(htmlClone, "img[srcset], source[srcset]"); + + return { + html: "\n" + htmlClone.outerHTML, + finalUrl: location.href, + }; +})() +`; + +export async function captureNormalizedPageSnapshot( + browser: BrowserSession, +): Promise { + return browser.evaluate(CAPTURE_NORMALIZED_PAGE_SCRIPT); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts new file mode 100644 index 0000000..986fb32 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts @@ -0,0 +1,201 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { spawnSync } from "node:child_process"; + +export interface ResolveSharedChromeProfileDirOptions { + envNames?: string[]; + appDataDirName?: string; + profileDirName?: string; +} + +export interface FindExistingChromeDebugPortOptions { + profileDir: string; + timeoutMs?: number; +} + +interface ChromeVersionResponse { + webSocketDebuggerUrl?: string; +} + +const CHROME_LOCK_FILE_NAMES = ["SingletonLock", "SingletonSocket", "SingletonCookie", "chrome.pid"] as const; + +function resolveDataBaseDir(): string { + if (process.platform === "darwin") { + return path.join(os.homedir(), "Library", "Application Support"); + } + if (process.platform === "win32") { + return process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"); + } + return process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"); +} + +export function resolveSharedChromeProfileDir( + options: ResolveSharedChromeProfileDirOptions = {}, +): string { + for (const envName of options.envNames ?? []) { + const override = process.env[envName]?.trim(); + if (override) { + return path.resolve(override); + } + } + + const appDataDirName = options.appDataDirName ?? "baoyu-skills"; + const profileDirName = options.profileDirName ?? "chrome-profile"; + return path.join(resolveDataBaseDir(), appDataDirName, profileDirName); +} + +export function resolveChromeProfileDir(profileDir?: string): string { + if (profileDir?.trim()) { + return path.resolve(profileDir.trim()); + } + + return resolveSharedChromeProfileDir({ + envNames: ["BAOYU_CHROME_PROFILE_DIR"], + appDataDirName: "baoyu-skills", + profileDirName: "chrome-profile", + }); +} + +export function ensureChromeProfileDir(profileDir: string): string { + fs.mkdirSync(profileDir, { recursive: true }); + return profileDir; +} + +export function hasChromeLockArtifacts(entries: readonly string[]): boolean { + return CHROME_LOCK_FILE_NAMES.some((name) => entries.includes(name)); +} + +export function shouldRetryChromeLaunchRecovery(options: { + hasLockArtifacts: boolean; + hasLiveOwner: boolean; +}): boolean { + return options.hasLockArtifacts && !options.hasLiveOwner; +} + +export function findChromeProcessUsingProfile(profileDir: string): boolean { + if (process.platform === "win32") { + return false; + } + + try { + const result = spawnSync("ps", ["aux"], { + encoding: "utf8", + timeout: 5_000, + }); + if (result.status !== 0 || !result.stdout) { + return false; + } + + return result.stdout + .split("\n") + .some((line) => line.includes(`--user-data-dir=${profileDir}`)); + } catch { + return false; + } +} + +export function cleanChromeLockArtifacts(profileDir: string): void { + for (const name of CHROME_LOCK_FILE_NAMES) { + try { + fs.unlinkSync(path.join(profileDir, name)); + } catch { + // Ignore missing files and continue cleaning the remaining artifacts. + } + } +} + +export async function listChromeProfileEntries(profileDir: string): Promise { + try { + return await fs.promises.readdir(profileDir); + } catch { + return []; + } +} + +async function fetchWithTimeout(url: string, timeoutMs = 3_000): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { + redirect: "follow", + signal: controller.signal, + }); + } finally { + clearTimeout(timer); + } +} + +async function fetchJson(url: string, timeoutMs = 3_000): Promise { + const response = await fetchWithTimeout(url, timeoutMs); + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + return (await response.json()) as T; +} + +async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise { + try { + const version = await fetchJson(`http://127.0.0.1:${port}/json/version`, timeoutMs); + return Boolean(version.webSocketDebuggerUrl); + } catch { + return false; + } +} + +function parseDevToolsActivePort(filePath: string): { port: number; wsPath: string } | null { + try { + const content = fs.readFileSync(filePath, "utf8"); + const lines = content.split(/\r?\n/); + const port = Number.parseInt(lines[0]?.trim() ?? "", 10); + const wsPath = lines[1]?.trim() ?? ""; + if (port > 0 && wsPath) { + return { port, wsPath }; + } + } catch { + // Ignore and fall back to process inspection. + } + + return null; +} + +export async function findExistingChromeDebugPort( + options: FindExistingChromeDebugPortOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? 3_000; + const activePort = parseDevToolsActivePort(path.join(options.profileDir, "DevToolsActivePort")); + if (activePort && await isDebugPortReady(activePort.port, timeoutMs)) { + return activePort.port; + } + + if (process.platform === "win32") { + return null; + } + + try { + const result = spawnSync("ps", ["aux"], { + encoding: "utf8", + timeout: 5_000, + }); + if (result.status !== 0 || !result.stdout) { + return null; + } + + const lines = result.stdout + .split("\n") + .filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port=")); + + for (const line of lines) { + const match = line.match(/--remote-debugging-port=(\d+)/); + const port = Number.parseInt(match?.[1] ?? "", 10); + if (port > 0 && await isDebugPortReady(port, timeoutMs)) { + return port; + } + } + } catch { + // Ignore and report no reusable debugger. + } + + return null; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts new file mode 100644 index 0000000..7f66e8a --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts @@ -0,0 +1,155 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { CdpClient, TargetSession, evaluateRuntime } from "./cdp-client"; + +interface NavigationResult { + errorText?: string; +} + +const execFileAsync = promisify(execFile); +const MACOS_BROWSER_APP_IDS = [ + "com.google.Chrome", + "org.chromium.Chromium", + "com.brave.Browser", + "com.microsoft.edgemac", +]; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function activateBrowserApp(): Promise { + if (process.platform !== "darwin") { + return; + } + + for (const appId of MACOS_BROWSER_APP_IDS) { + try { + await execFileAsync("osascript", ["-e", `tell application id "${appId}" to activate`]); + return; + } catch { + // Try the next installed browser bundle id. + } + } +} + +export class BrowserSession { + private constructor( + private readonly cdp: CdpClient, + public readonly targetSession: TargetSession, + public readonly interactive: boolean, + ) {} + + static async open( + cdp: CdpClient, + options: { + initialUrl?: string; + interactive?: boolean; + } = {}, + ): Promise { + const targetSession = await cdp.createPageSession({ + initialUrl: options.initialUrl, + visible: options.interactive, + }); + const browser = new BrowserSession(cdp, targetSession, Boolean(options.interactive)); + if (browser.interactive) { + await browser.bringToFront().catch(() => {}); + } + return browser; + } + + async goto(url: string, timeoutMs = 30_000): Promise { + const loadPromise = this.targetSession.waitForEvent("Page.loadEventFired", undefined, timeoutMs).catch(() => null); + const result = await this.targetSession.send("Page.navigate", { url }); + if (result.errorText) { + throw new Error(`Navigation failed: ${result.errorText}`); + } + await loadPromise; + await this.waitForReadyState(timeoutMs); + } + + async waitForReadyState(timeoutMs = 30_000): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const state = await this.evaluate("document.readyState"); + if (state === "interactive" || state === "complete") { + return; + } + await sleep(150); + } + throw new Error("Timed out waiting for document.readyState"); + } + + async evaluate(expression: string): Promise { + return evaluateRuntime(this.targetSession, expression); + } + + async getHTML(): Promise { + return this.evaluate("document.documentElement.outerHTML"); + } + + async getTitle(): Promise { + return this.evaluate("document.title"); + } + + async getURL(): Promise { + return this.evaluate("window.location.href"); + } + + async bringToFront(): Promise { + await this.targetSession.send("Page.bringToFront").catch(async () => { + await this.cdp.sendBrowserCommand("Target.activateTarget", { + targetId: this.targetSession.targetId, + }); + }); + if (this.interactive) { + await activateBrowserApp().catch(() => {}); + } + } + + async click(selector: string): Promise { + const result = await this.evaluate<{ ok: boolean; error?: string }>(` + (() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) { + return { ok: false, error: "Element not found" }; + } + element.scrollIntoView({ block: "center", inline: "center" }); + if (element instanceof HTMLElement) { + element.click(); + return { ok: true }; + } + return { ok: false, error: "Element is not clickable" }; + })() + `); + + if (!result.ok) { + throw new Error(result.error ?? `Failed to click ${selector}`); + } + } + + async scrollToEnd(options: { stepPx?: number; delayMs?: number; maxSteps?: number } = {}): Promise { + const stepPx = options.stepPx ?? 1_400; + const delayMs = options.delayMs ?? 250; + const maxSteps = options.maxSteps ?? 6; + + for (let step = 0; step < maxSteps; step += 1) { + const done = await this.evaluate(` + (() => { + const before = window.scrollY; + window.scrollBy(0, ${stepPx}); + const atBottom = window.innerHeight + window.scrollY >= document.body.scrollHeight - 4; + return atBottom || window.scrollY === before; + })() + `); + if (done) { + break; + } + await sleep(delayMs); + } + } + + async close(): Promise { + await this.cdp.closeTarget(this.targetSession.targetId); + } +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/cli.ts b/skills/baoyu-url-to-markdown/scripts/lib/cli.ts new file mode 100755 index 0000000..abde93f --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/cli.ts @@ -0,0 +1,227 @@ +#!/usr/bin/env bun + +import { + runConvertCommand, + type ConvertCommandOptions, + type OutputFormat, + type WaitMode, +} from "./commands/convert"; + +export const HELP_TEXT = ` +baoyu-fetch - Read a URL into Markdown or JSON with Chrome CDP + +Usage: + baoyu-fetch [options] + +Options: + --output Save output to file + --format Output format: markdown | json + --json Alias for --format json + --adapter Force an adapter (e.g. x, generic) + --download-media Download adapter-reported media into ./imgs and ./videos, then rewrite markdown links + --media-dir

Base directory for downloaded media. Defaults to the output directory + --debug-dir Write debug artifacts + --cdp-url Reuse an existing Chrome DevTools endpoint + --browser-path Explicit Chrome binary path + --chrome-profile-dir + Chrome user data dir. Defaults to BAOYU_CHROME_PROFILE_DIR + or baoyu-skills/chrome-profile. + --headless Launch a temporary headless Chrome if needed + --wait-for Wait mode: interaction | force + interaction: start visible Chrome and auto-wait only when login or verification is required + force: start visible Chrome, then auto-continue after it detects login/challenge progress + or continue immediately when you press Enter + --wait-for-interaction + Alias for --wait-for interaction + --wait-for-login Alias for --wait-for interaction + --interaction-timeout + How long to wait for manual interaction before failing (default: 600000) + --interaction-poll-interval + How often to poll interaction state while waiting (default: 1500) + --login-timeout Alias for --interaction-timeout + --login-poll-interval + Alias for --interaction-poll-interval + --timeout Page timeout in milliseconds (default: 30000) + --help Show help + +Examples: + baoyu-fetch https://example.com + baoyu-fetch https://example.com --format markdown --output article.md --download-media + baoyu-fetch https://example.com --format json --output article.json + baoyu-fetch https://x.com/lennysan/status/2036483059407810640 --wait-for interaction + baoyu-fetch https://x.com/lennysan/status/2036483059407810640 --wait-for force +`.trim(); + +interface CliOptions extends ConvertCommandOptions { + url?: string; + help: boolean; +} + +function normalizeWaitMode(raw: string): WaitMode { + const value = raw.toLowerCase(); + if (value === "interaction" || value === "auto") { + return "interaction"; + } + if (value === "force" || value === "manual" || value === "always") { + return "force"; + } + throw new Error(`Invalid wait mode: ${raw}. Expected interaction or force.`); +} + +function normalizeOutputFormat(raw: string): OutputFormat { + const value = raw.toLowerCase(); + if (value === "markdown" || value === "json") { + return value; + } + + throw new Error(`Invalid output format: ${raw}. Expected markdown or json.`); +} + +export function parseArgs(argv: string[]): CliOptions { + const options: CliOptions = { + format: "markdown", + headless: false, + downloadMedia: false, + waitMode: "none", + interactionTimeoutMs: 600_000, + interactionPollIntervalMs: 1_500, + timeoutMs: 30_000, + help: false, + }; + + const args = argv.slice(2); + for (let index = 0; index < args.length; index += 1) { + const value = args[index]; + + if (value === "--help" || value === "-h") { + options.help = true; + continue; + } + if (value === "--format") { + const format = args[index + 1]; + if (!format) { + throw new Error("--format requires a value"); + } + options.format = normalizeOutputFormat(format); + index += 1; + continue; + } + if (value === "--json") { + options.format = "json"; + continue; + } + if (value === "--download-media") { + options.downloadMedia = true; + continue; + } + if (value === "--headless") { + options.headless = true; + continue; + } + if (value === "--wait-for") { + const mode = args[index + 1]; + if (!mode) { + throw new Error("--wait-for requires a mode"); + } + options.waitMode = normalizeWaitMode(mode); + index += 1; + continue; + } + if (value === "--wait-for-interaction" || value === "--wait-for-login") { + options.waitMode = "interaction"; + continue; + } + if (value === "--output") { + options.output = args[index + 1]; + index += 1; + continue; + } + if (value === "--adapter") { + options.adapter = args[index + 1]; + index += 1; + continue; + } + if (value === "--debug-dir") { + options.debugDir = args[index + 1]; + index += 1; + continue; + } + if (value === "--media-dir") { + options.mediaDir = args[index + 1]; + index += 1; + continue; + } + if (value === "--cdp-url") { + options.cdpUrl = args[index + 1]; + index += 1; + continue; + } + if (value === "--browser-path") { + options.browserPath = args[index + 1]; + index += 1; + continue; + } + if (value === "--chrome-profile-dir") { + options.chromeProfileDir = args[index + 1]; + index += 1; + continue; + } + if (value === "--timeout") { + const parsed = Number(args[index + 1]); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Invalid timeout: ${args[index + 1]}`); + } + options.timeoutMs = parsed; + index += 1; + continue; + } + if (value === "--interaction-timeout" || value === "--login-timeout") { + const parsed = Number(args[index + 1]); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Invalid interaction timeout: ${args[index + 1]}`); + } + options.interactionTimeoutMs = parsed; + index += 1; + continue; + } + if (value === "--interaction-poll-interval" || value === "--login-poll-interval") { + const parsed = Number(args[index + 1]); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Invalid interaction poll interval: ${args[index + 1]}`); + } + options.interactionPollIntervalMs = parsed; + index += 1; + continue; + } + if (value.startsWith("-")) { + throw new Error(`Unknown option: ${value}`); + } + if (!options.url) { + options.url = value; + continue; + } + throw new Error(`Unexpected argument: ${value}`); + } + + return options; +} + +async function main(): Promise { + try { + const options = parseArgs(process.argv); + if (options.help || !options.url) { + console.log(HELP_TEXT); + return; + } + + await runConvertCommand(options); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = 1; + } +} + +if (import.meta.main) { + void main(); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts b/skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts new file mode 100644 index 0000000..659e18e --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts @@ -0,0 +1,580 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { connectChrome, type ChromeConnection } from "../browser/chrome-launcher"; +import { CdpClient } from "../browser/cdp-client"; +import { detectInteractionGate } from "../browser/interaction-gates"; +import { NetworkJournal } from "../browser/network-journal"; +import { BrowserSession } from "../browser/session"; +import { genericAdapter, resolveAdapter } from "../adapters"; +import { isXSessionReady } from "../adapters/x/session"; +import type { ExtractedDocument } from "../extract/document"; +import { renderMarkdown } from "../extract/markdown-renderer"; +import { downloadMediaAssets } from "../media/default-downloader"; +import { rewriteMarkdownMediaLinks } from "../media/markdown-media"; +import { createLogger } from "../utils/logger"; +import { normalizeUrl } from "../utils/url"; +import type { + Adapter, + AdapterContext, + AdapterLoginInfo, + LoginState, + MediaAsset, + WaitForInteractionRequest, +} from "../adapters/types"; + +export type WaitMode = "none" | "interaction" | "force"; +export type OutputFormat = "markdown" | "json"; + +export interface ConvertCommandOptions { + url?: string; + output?: string; + format: OutputFormat; + adapter?: string; + debugDir?: string; + cdpUrl?: string; + browserPath?: string; + chromeProfileDir?: string; + headless: boolean; + downloadMedia: boolean; + mediaDir?: string; + waitMode: WaitMode; + interactionTimeoutMs: number; + interactionPollIntervalMs: number; + timeoutMs: number; +} + +interface RuntimeResources { + chrome: ChromeConnection; + cdp: CdpClient; + browser: BrowserSession; + network: NetworkJournal; + interactive: boolean; +} + +interface ForceWaitSnapshot { + url: string; + hasGate: boolean; + loginState: LoginState | "unavailable"; + sessionReady: boolean; +} + +interface SuccessfulConvertOutput { + adapter: string; + status: "ok"; + login?: AdapterLoginInfo; + media: MediaAsset[]; + downloads: Awaited> | null; + document: ExtractedDocument; + markdown: string; +} + +interface InteractionRequiredOutput { + adapter: string; + status: "needs_interaction"; + login?: AdapterLoginInfo; + interaction: WaitForInteractionRequest; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isForceWaitSessionReady(snapshot: ForceWaitSnapshot): boolean { + return snapshot.sessionReady; +} + +export function shouldKeepBrowserOpenAfterInteraction(options: { + launched: boolean; + interaction: Pick; +}): boolean { + return options.launched && options.interaction.kind === "login" && options.interaction.provider === "x"; +} + +export function shouldAutoContinueForceWait( + initial: ForceWaitSnapshot, + current: ForceWaitSnapshot, +): boolean { + if (initial.hasGate && !current.hasGate) { + return true; + } + + if (initial.loginState === "logged_out" && current.loginState !== "logged_out" && isForceWaitSessionReady(current)) { + return true; + } + + if (initial.loginState !== "logged_in" && current.loginState === "logged_in" && isForceWaitSessionReady(current)) { + return true; + } + + if ( + current.url !== initial.url && + !current.hasGate && + current.loginState !== "logged_out" && + isForceWaitSessionReady(current) + ) { + return true; + } + + return false; +} + +async function writeOutput(path: string, content: string): Promise { + const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : ""; + if (directory) { + await mkdir(directory, { recursive: true }); + } + await writeFile(path, content, "utf8"); +} + +async function writeDebugArtifacts( + debugDir: string, + document: ExtractedDocument, + markdown: string, + browser: BrowserSession, + network: NetworkJournal, +): Promise { + await mkdir(debugDir, { recursive: true }); + + const html = await browser.getHTML().catch(() => ""); + const networkDump = await network.toJSON({ includeBodies: true }); + + await Promise.all([ + writeFile(join(debugDir, "document.json"), JSON.stringify(document, null, 2), "utf8"), + writeFile(join(debugDir, "markdown.md"), markdown, "utf8"), + writeFile(join(debugDir, "page.html"), html, "utf8"), + writeFile(join(debugDir, "network.json"), JSON.stringify(networkDump, null, 2), "utf8"), + ]); +} + +async function openRuntime( + options: ConvertCommandOptions, + interactive: boolean, + debugEnabled: boolean, +): Promise { + const logger = createLogger(debugEnabled); + if (interactive) { + logger.info("Opening Chrome in interactive mode."); + } + const chrome = await connectChrome({ + cdpUrl: options.cdpUrl, + browserPath: options.browserPath, + profileDir: options.chromeProfileDir, + headless: interactive ? false : options.headless, + logger, + }); + + const cdp = await CdpClient.connect(chrome.browserWsUrl); + const browser = await BrowserSession.open(cdp, { interactive }); + if (interactive) { + await browser.bringToFront().catch(() => {}); + } + const network = new NetworkJournal(browser.targetSession, logger); + await network.start(); + + return { + chrome, + cdp, + browser, + network, + interactive, + }; +} + +async function closeRuntime(runtime: RuntimeResources | null | undefined): Promise { + if (!runtime) { + return; + } + runtime.network.stop(); + await runtime.browser.close().catch(() => {}); + await runtime.cdp.close().catch(() => {}); + await runtime.chrome.close().catch(() => {}); +} + +async function isInteractionSessionReady( + context: AdapterContext, + interaction: WaitForInteractionRequest, +): Promise { + if (interaction.provider !== "x") { + return true; + } + return await isXSessionReady(context).catch(() => false); +} + +async function reopenInteractiveRuntime( + runtime: RuntimeResources, + options: ConvertCommandOptions, + debugEnabled: boolean, +): Promise { + if (runtime.interactive) { + return runtime; + } + + await closeRuntime(runtime); + return openRuntime(options, true, debugEnabled); +} + +async function captureForceWaitSnapshot( + adapter: Adapter, + context: AdapterContext, +): Promise { + const [gate, url, login] = await Promise.all([ + detectInteractionGate(context.browser).catch(() => null), + context.browser.getURL().catch(() => context.input.url.toString()), + adapter.checkLogin?.(context).catch(() => ({ + provider: adapter.name, + state: "unknown" as const, + })), + ]); + + return { + url, + hasGate: Boolean(gate), + loginState: login?.state ?? "unavailable", + sessionReady: adapter.name === "x" ? await isXSessionReady(context).catch(() => false) : true, + }; +} + +async function waitForForceResume( + adapter: Adapter, + context: AdapterContext, + options: ConvertCommandOptions, +): Promise { + if (context.interactive) { + await context.browser.bringToFront().catch(() => {}); + } + + const prompt = + "Chrome is ready. Complete any manual login or verification. Extraction will continue automatically after it detects progress, or press Enter to continue immediately."; + context.log.info(prompt); + + const rl = createInterface({ + input: process.stdin, + output: process.stderr, + }); + + let manualContinue = false; + let closed = false; + const closeReadline = (): void => { + if (!closed) { + closed = true; + rl.close(); + } + }; + + rl.once("line", () => { + manualContinue = true; + closeReadline(); + }); + + const initial = await captureForceWaitSnapshot(adapter, context); + const startedAt = Date.now(); + + try { + while (Date.now() - startedAt < options.interactionTimeoutMs) { + if (manualContinue) { + return; + } + + const current = await captureForceWaitSnapshot(adapter, context); + if (shouldAutoContinueForceWait(initial, current)) { + return; + } + + await sleep(options.interactionPollIntervalMs); + } + } finally { + closeReadline(); + } + + throw new Error("Timed out waiting for force-mode interaction to complete"); +} + +async function waitForInteraction( + adapter: Adapter, + context: AdapterContext, + interaction: WaitForInteractionRequest, + options: ConvertCommandOptions, +): Promise { + const timeoutMs = interaction.timeoutMs ?? options.interactionTimeoutMs; + const pollIntervalMs = interaction.pollIntervalMs ?? options.interactionPollIntervalMs; + if (context.interactive) { + await context.browser.bringToFront().catch(() => {}); + } + context.log.info(interaction.prompt); + + const startedAt = Date.now(); + let lastLogin: AdapterLoginInfo | null = null; + + while (Date.now() - startedAt < timeoutMs) { + if (interaction.kind === "login" && adapter.checkLogin) { + lastLogin = await adapter.checkLogin(context); + if (lastLogin.state === "logged_in" && await isInteractionSessionReady(context, interaction)) { + return lastLogin; + } + } + + const gate = await detectInteractionGate(context.browser); + if (!gate) { + if (interaction.kind !== "login") { + return lastLogin ?? { + provider: interaction.provider, + state: "unknown", + reason: `${interaction.provider} challenge cleared`, + }; + } + + if (!adapter.checkLogin) { + return { + provider: interaction.provider, + state: "unknown", + }; + } + + lastLogin = await adapter.checkLogin(context); + if (lastLogin.state !== "logged_out" && await isInteractionSessionReady(context, interaction)) { + return lastLogin; + } + } + await sleep(pollIntervalMs); + } + + const reason = lastLogin?.reason ? ` (${lastLogin.reason})` : ""; + throw new Error(`Timed out waiting for ${interaction.provider} interaction${reason}`); +} + +export function formatOutputContent( + format: OutputFormat, + payload: SuccessfulConvertOutput | InteractionRequiredOutput, +): string { + if (format === "json") { + return JSON.stringify(payload, null, 2); + } + + if (payload.status !== "ok") { + throw new Error("Markdown output is only available for successful extraction results"); + } + + return payload.markdown; +} + +function printOutput(content: string): void { + process.stdout.write(content); + if (!content.endsWith("\n")) { + process.stdout.write("\n"); + } +} + +export async function runConvertCommand(options: ConvertCommandOptions): Promise { + if (!options.url) { + throw new Error("URL is required"); + } + if (options.downloadMedia && !options.output) { + throw new Error("--download-media requires --output so media paths can be rewritten relative to the saved output file"); + } + + const url = normalizeUrl(options.url); + let runtime = await openRuntime(options, options.waitMode !== "none", Boolean(options.debugDir)); + const logger = createLogger(Boolean(options.debugDir)); + let didLogin = false; + let adapter: Adapter | null = null; + let context: AdapterContext | null = null; + + try { + adapter = resolveAdapter({ url }, options.adapter); + context = { + input: { url }, + browser: runtime.browser, + network: runtime.network, + cdp: runtime.cdp, + log: logger, + outputFormat: options.format, + timeoutMs: options.timeoutMs, + interactive: runtime.interactive, + downloadMedia: options.downloadMedia, + }; + + if (adapter.restoreCookies) { + const restored = await adapter.restoreCookies(context, runtime.chrome.profileDir).catch(() => false); + if (restored) logger.info(`Restored ${adapter.name} session cookies from sidecar.`); + } + + if (options.waitMode === "interaction" && adapter.checkLogin) { + await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {}); + const preLogin = await adapter.checkLogin(context); + if (preLogin.state !== "logged_in") { + didLogin = true; + await waitForInteraction(adapter, context, { + type: "wait_for_interaction", + kind: "login", + provider: preLogin.provider ?? adapter.name, + prompt: `Please sign in to ${adapter.name === "x" ? "X" : adapter.name} in the opened Chrome window. Extraction will continue automatically once login is detected.`, + reason: preLogin.reason ?? `Not logged in to ${adapter.name}`, + requiresVisibleBrowser: true, + }, options); + } + } + + if (options.waitMode === "force") { + await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {}); + await waitForForceResume(adapter, context, options); + } + + let result = await adapter.process(context); + + if (result.status === "no_document") { + const interaction = await detectInteractionGate(context.browser); + if (interaction) { + result = { + status: "needs_interaction", + interaction, + login: result.login, + }; + } + } + + while (result.status === "needs_interaction") { + if (options.waitMode === "none") { + if (options.format === "json") { + printOutput( + formatOutputContent(options.format, { + adapter: adapter.name, + status: result.status, + login: result.login, + interaction: result.interaction, + }), + ); + return; + } + + throw new Error(`${adapter.name} requires manual interaction. Re-run with --wait-for interaction to continue after completing it.`); + } + + if (result.interaction.requiresVisibleBrowser !== false) { + runtime = await reopenInteractiveRuntime(runtime, options, Boolean(options.debugDir)); + } + + context = { + input: { url }, + browser: runtime.browser, + network: runtime.network, + cdp: runtime.cdp, + log: logger, + outputFormat: options.format, + timeoutMs: options.timeoutMs, + interactive: runtime.interactive, + downloadMedia: options.downloadMedia, + }; + + await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {}); + if (result.interaction.kind === "login") { + didLogin = true; + } + await waitForInteraction(adapter, context, result.interaction, options); + result = await adapter.process(context); + + if (result.status === "no_document") { + const interaction = await detectInteractionGate(context.browser); + if (interaction) { + result = { + status: "needs_interaction", + interaction, + login: result.login, + }; + } + } + } + + let document: ExtractedDocument | null = result.status === "ok" ? result.document : null; + let media: MediaAsset[] = result.status === "ok" ? (result.media ?? []) : []; + let login = result.login; + let mediaAdapter = adapter; + + if (!document && adapter.name !== genericAdapter.name && result.status === "no_document") { + logger.info(`Adapter ${adapter.name} returned no structured document; falling back to generic extraction`); + const fallback = await genericAdapter.process(context); + if (fallback.status === "ok") { + document = fallback.document; + media = fallback.media ?? []; + mediaAdapter = genericAdapter; + } + } + + if (!document) { + throw new Error("Failed to extract a document from the target URL"); + } + + document.requestedUrl ??= url.toString(); + + let markdown = renderMarkdown(document); + let downloadResult: + | Awaited> + | null = null; + + if (options.downloadMedia && options.output) { + downloadResult = mediaAdapter.downloadMedia + ? await mediaAdapter.downloadMedia({ + media, + outputPath: options.output, + mediaDir: options.mediaDir, + log: logger, + }) + : await downloadMediaAssets({ + media, + outputPath: options.output, + mediaDir: options.mediaDir, + log: logger, + }); + + markdown = rewriteMarkdownMediaLinks(markdown, downloadResult.replacements); + if (downloadResult.downloadedImages > 0 || downloadResult.downloadedVideos > 0) { + logger.info( + `Downloaded ${downloadResult.downloadedImages} images and ${downloadResult.downloadedVideos} videos`, + ); + } + } + + if (options.output) { + await writeOutput( + options.output, + formatOutputContent(options.format, { + adapter: document.adapter ?? adapter.name, + status: "ok", + login, + media, + downloads: downloadResult, + document, + markdown, + }), + ); + logger.info(`Saved ${options.format} to ${options.output}`); + } + + if (options.debugDir) { + await writeDebugArtifacts(options.debugDir, document, markdown, runtime.browser, runtime.network); + logger.info(`Wrote debug artifacts to ${options.debugDir}`); + } + + if (options.format === "json") { + printOutput( + formatOutputContent(options.format, { + adapter: document.adapter ?? adapter.name, + status: "ok", + login, + media, + downloads: downloadResult, + document, + markdown, + }), + ); + return; + } + + printOutput(markdown); + } finally { + if (adapter?.exportCookies && context) { + await adapter.exportCookies(context, runtime.chrome.profileDir).catch(() => {}); + } + await closeRuntime(runtime); + } +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts new file mode 100644 index 0000000..baaee51 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts @@ -0,0 +1,51 @@ +export type ContentBlock = + | { + type: "paragraph"; + text: string; + } + | { + type: "heading"; + depth: number; + text: string; + } + | { + type: "list"; + ordered: boolean; + items: string[]; + } + | { + type: "quote"; + text: string; + } + | { + type: "code"; + code: string; + language?: string; + } + | { + type: "image"; + url: string; + alt?: string; + } + | { + type: "html"; + html: string; + } + | { + type: "markdown"; + markdown: string; + }; + +export interface ExtractedDocument { + url: string; + requestedUrl?: string; + canonicalUrl?: string; + title?: string; + author?: string; + siteName?: string; + publishedAt?: string; + summary?: string; + content: ContentBlock[]; + metadata?: Record; + adapter?: string; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts new file mode 100644 index 0000000..f7034be --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts @@ -0,0 +1,467 @@ +import { JSDOM } from "jsdom"; + +export interface CleanHtmlOptions { + removeAds?: boolean; + removeBase64Images?: boolean; + onlyMainContent?: boolean; + includeSelectors?: string[]; + excludeSelectors?: string[]; +} + +const ALWAYS_REMOVE_SELECTORS = [ + "script", + "style", + "noscript", + "link[rel='stylesheet']", + "[hidden]", + "[aria-hidden='true']", + "[style*='display: none']", + "[style*='display:none']", + "[style*='visibility: hidden']", + "[style*='visibility:hidden']", + "svg[aria-hidden='true']", + "svg.icon", + "svg[class*='icon']", + "template", + "meta", + "iframe", + "canvas", + "object", + "embed", + "form", + "input", + "select", + "textarea", + "button", +]; + +const OVERLAY_SELECTORS = [ + "[class*='modal']", + "[class*='popup']", + "[class*='overlay']", + "[class*='dialog']", + "[role='dialog']", + "[role='alertdialog']", + "[class*='cookie']", + "[class*='consent']", + "[class*='gdpr']", + "[class*='privacy-banner']", + "[class*='notification-bar']", + "[id*='cookie']", + "[id*='consent']", + "[id*='gdpr']", + "[style*='position: fixed']", + "[style*='position:fixed']", + "[style*='position: sticky']", + "[style*='position:sticky']", +]; + +const NAVIGATION_SELECTORS = [ + "header", + "footer", + "nav", + "aside", + ".header", + ".top", + ".navbar", + "#header", + ".footer", + ".bottom", + "#footer", + ".sidebar", + ".side", + ".aside", + "#sidebar", + ".modal", + ".popup", + "#modal", + ".overlay", + ".ad", + ".ads", + ".advert", + "#ad", + ".lang-selector", + ".language", + "#language-selector", + ".social", + ".social-media", + ".social-links", + "#social", + ".menu", + ".navigation", + "#nav", + ".breadcrumbs", + "#breadcrumbs", + ".share", + "#share", + ".widget", + "#widget", + ".cookie", + "#cookie", +]; + +const FORCE_INCLUDE_SELECTORS = [ + "#main", + "#content", + "#main-content", + "#article", + "#post", + "#page-content", + "main", + "article", + "[role='main']", + ".main-content", + ".content", + ".post-content", + ".article-content", + ".entry-content", + ".page-content", + ".article-body", + ".post-body", + ".story-content", + ".blog-content", +]; + +const AD_SELECTORS = [ + "ins.adsbygoogle", + ".google-ad", + ".adsense", + "[data-ad]", + "[data-ads]", + "[data-ad-slot]", + "[data-ad-client]", + ".ad-container", + ".ad-wrapper", + ".advertisement", + ".sponsored-content", + "img[width='1'][height='1']", + "img[src*='pixel']", + "img[src*='tracking']", + "img[src*='analytics']", +]; + +function getLinkDensity(element: Element): number { + const text = element.textContent || ""; + const textLength = text.trim().length; + if (textLength === 0) { + return 1; + } + + let linkLength = 0; + element.querySelectorAll("a").forEach((link) => { + linkLength += (link.textContent || "").trim().length; + }); + + return linkLength / textLength; +} + +function getContentScore(element: Element): number { + let score = 0; + const text = element.textContent || ""; + const textLength = text.trim().length; + + score += Math.min(textLength / 100, 50); + score += element.querySelectorAll("p").length * 3; + score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2; + score += element.querySelectorAll("img").length; + + score -= element.querySelectorAll("a").length * 0.5; + score -= element.querySelectorAll("li").length * 0.2; + + const linkDensity = getLinkDensity(element); + if (linkDensity > 0.5) { + score -= 30; + } else if (linkDensity > 0.3) { + score -= 15; + } + + const className = typeof element.className === "string" ? element.className : ""; + const classAndId = `${className} ${element.id || ""}`; + if (/article|content|post|body|main|entry/i.test(classAndId)) { + score += 25; + } + if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) { + score -= 25; + } + + return score; +} + +function looksLikeNavigation(element: Element): boolean { + const linkDensity = getLinkDensity(element); + if (linkDensity > 0.5) { + return true; + } + + const listItems = element.querySelectorAll("li"); + const links = element.querySelectorAll("a"); + return listItems.length > 5 && links.length > listItems.length * 0.8; +} + +function removeElements(document: Document, selectors: string[]): void { + for (const selector of selectors) { + try { + document.querySelectorAll(selector).forEach((element) => element.remove()); + } catch { + // Ignore unsupported selectors. + } + } +} + +function removeWithProtection( + document: Document, + selectorsToRemove: string[], + protectedSelectors: string[], +): void { + for (const selector of selectorsToRemove) { + try { + document.querySelectorAll(selector).forEach((element) => { + const isProtected = protectedSelectors.some((protectedSelector) => { + try { + return element.matches(protectedSelector); + } catch { + return false; + } + }); + + if (isProtected) { + return; + } + + const containsProtected = protectedSelectors.some((protectedSelector) => { + try { + return element.querySelector(protectedSelector) !== null; + } catch { + return false; + } + }); + + if (containsProtected) { + return; + } + + element.remove(); + }); + } catch { + // Ignore unsupported selectors. + } + } +} + +function isValidContent(element: Element | null): element is Element { + if (!element) { + return false; + } + const text = element.textContent || ""; + if (text.trim().length < 100) { + return false; + } + return !looksLikeNavigation(element); +} + +function findMainContent(document: Document): Element | null { + const main = document.querySelector("main"); + if (isValidContent(main) && getLinkDensity(main) < 0.4) { + return main; + } + + const roleMain = document.querySelector('[role="main"]'); + if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) { + return roleMain; + } + + const articles = document.querySelectorAll("article"); + if (articles.length === 1 && isValidContent(articles[0] ?? null)) { + return articles[0] ?? null; + } + + const contentSelectors = [ + "#content", + "#main-content", + "#main", + ".content", + ".main-content", + ".post-content", + ".article-content", + ".entry-content", + ".page-content", + ".article-body", + ".post-body", + ".story-content", + ".blog-content", + ]; + + for (const selector of contentSelectors) { + try { + const element = document.querySelector(selector); + if (isValidContent(element) && getLinkDensity(element) < 0.4) { + return element; + } + } catch { + // Ignore invalid selectors. + } + } + + const candidates: Array<{ element: Element; score: number }> = []; + document.querySelectorAll("div, section, article").forEach((element) => { + const text = element.textContent || ""; + if (text.trim().length < 200) { + return; + } + + const score = getContentScore(element); + if (score > 0) { + candidates.push({ element, score }); + } + }); + + candidates.sort((left, right) => right.score - left.score); + if ((candidates[0]?.score ?? 0) > 20) { + return candidates[0]?.element ?? null; + } + + return null; +} + +function removeBase64ImagesFromDocument(document: Document): void { + document.querySelectorAll("img[src^='data:']").forEach((element) => element.remove()); + + document.querySelectorAll("[style*='data:image']").forEach((element) => { + const style = element.getAttribute("style"); + if (!style) { + return; + } + + const cleanedStyle = style.replace( + /background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi, + "", + ); + + if (cleanedStyle.trim()) { + element.setAttribute("style", cleanedStyle); + } else { + element.removeAttribute("style"); + } + }); + + document + .querySelectorAll("source[src^='data:'], source[srcset*='data:']") + .forEach((element) => element.remove()); +} + +function makeAbsoluteUrl(value: string, baseUrl: string): string | null { + try { + return new URL(value, baseUrl).toString(); + } catch { + return null; + } +} + +function convertRelativeUrls(document: Document, baseUrl: string): void { + document.querySelectorAll("[src]").forEach((element) => { + const src = element.getAttribute("src"); + if (!src || src.startsWith("http") || src.startsWith("//") || src.startsWith("data:")) { + return; + } + + const absolute = makeAbsoluteUrl(src, baseUrl); + if (absolute) { + element.setAttribute("src", absolute); + } + }); + + document.querySelectorAll("[href]").forEach((element) => { + const href = element.getAttribute("href"); + if ( + !href || + href.startsWith("http") || + href.startsWith("//") || + href.startsWith("#") || + href.startsWith("mailto:") || + href.startsWith("tel:") || + href.startsWith("javascript:") + ) { + return; + } + + const absolute = makeAbsoluteUrl(href, baseUrl); + if (absolute) { + element.setAttribute("href", absolute); + } + }); +} + +function removeComments(document: Document): void { + const walker = document.createTreeWalker(document, document.defaultView?.NodeFilter.SHOW_COMMENT ?? 128); + const comments: Comment[] = []; + while (walker.nextNode()) { + comments.push(walker.currentNode as Comment); + } + comments.forEach((comment) => comment.parentNode?.removeChild(comment)); +} + +export function cleanHtml( + html: string, + baseUrl: string, + options: CleanHtmlOptions = {}, +): string { + const { + removeAds = true, + removeBase64Images = true, + onlyMainContent = true, + includeSelectors, + excludeSelectors, + } = options; + + const dom = new JSDOM(html, { url: baseUrl }); + const { document } = dom.window; + + removeElements(document, ALWAYS_REMOVE_SELECTORS); + removeElements(document, OVERLAY_SELECTORS); + + if (removeAds) { + removeElements(document, AD_SELECTORS); + } + + if (excludeSelectors?.length) { + removeElements(document, excludeSelectors); + } + + if (onlyMainContent) { + removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS); + + const mainContent = findMainContent(document); + if (mainContent && document.body) { + const clone = mainContent.cloneNode(true); + document.body.innerHTML = ""; + document.body.appendChild(clone); + } + } + + if (includeSelectors?.length && document.body) { + const matchedElements: Element[] = []; + for (const selector of includeSelectors) { + try { + document.querySelectorAll(selector).forEach((element) => { + matchedElements.push(element.cloneNode(true) as Element); + }); + } catch { + // Ignore invalid selectors. + } + } + + if (matchedElements.length > 0) { + document.body.innerHTML = ""; + matchedElements.forEach((element) => document.body?.appendChild(element)); + } + } + + if (removeBase64Images) { + removeBase64ImagesFromDocument(document); + } + + removeComments(document); + convertRelativeUrls(document, baseUrl); + + return document.documentElement.outerHTML || html; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts new file mode 100644 index 0000000..cbd7ef4 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts @@ -0,0 +1,83 @@ +import { Readability } from "@mozilla/readability"; +import { JSDOM } from "jsdom"; +import type { ExtractedDocument } from "./document"; + +function getMetaContent(document: Document, selectors: string[]): string | undefined { + for (const selector of selectors) { + const value = document.querySelector(selector)?.getAttribute("content")?.trim(); + if (value) { + return value; + } + } + return undefined; +} + +export function extractDocumentFromHtml(input: { + url: string; + html: string; + adapter?: string; +}): ExtractedDocument { + const dom = new JSDOM(input.html, { url: input.url }); + const document = dom.window.document; + + const canonicalUrl = + document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ?? + getMetaContent(document, ['meta[property="og:url"]']); + + const siteName = getMetaContent(document, [ + 'meta[property="og:site_name"]', + 'meta[name="application-name"]', + ]); + + const metadataAuthor = getMetaContent(document, [ + 'meta[name="author"]', + 'meta[property="article:author"]', + 'meta[name="twitter:creator"]', + ]); + + const publishedAt = getMetaContent(document, [ + 'meta[property="article:published_time"]', + 'meta[name="pubdate"]', + 'meta[name="date"]', + 'meta[itemprop="datePublished"]', + ]); + + const article = new Readability(document).parse(); + const title = + article?.title?.trim() || + getMetaContent(document, ['meta[property="og:title"]']) || + document.title.trim() || + undefined; + + const summary = + article?.excerpt?.trim() || + getMetaContent(document, [ + 'meta[name="description"]', + 'meta[property="og:description"]', + 'meta[name="twitter:description"]', + ]); + + const contentHtml = + article?.content?.trim() || + document.querySelector("main")?.innerHTML?.trim() || + document.body?.innerHTML?.trim() || + ""; + + const author = article?.byline?.trim() || metadataAuthor; + + return { + url: input.url, + canonicalUrl, + title, + author, + siteName, + publishedAt, + summary, + adapter: input.adapter ?? "generic", + metadata: { + language: document.documentElement.lang || undefined, + }, + content: contentHtml ? [{ type: "html", html: contentHtml }] : [], + }; +} + diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts new file mode 100644 index 0000000..56aad98 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts @@ -0,0 +1,758 @@ +import { Readability } from "@mozilla/readability"; +import { Defuddle } from "defuddle/node"; +import { JSDOM, VirtualConsole } from "jsdom"; +import TurndownService from "turndown"; +import { gfm } from "turndown-plugin-gfm"; +import { collectMediaFromMarkdown } from "../media/markdown-media"; +import type { MediaAsset } from "../media/types"; +import { cleanHtml } from "./html-cleaner"; + +export interface HtmlConversionMetadata { + url: string; + canonicalUrl?: string; + siteName?: string; + title?: string; + summary?: string; + author?: string; + publishedAt?: string; + coverImage?: string; + language?: string; + capturedAt: string; +} + +export interface ConvertHtmlToMarkdownOptions { + enableRemoteMarkdownFallback?: boolean; + preserveBase64Images?: boolean; +} + +export interface HtmlToMarkdownResult { + metadata: HtmlConversionMetadata; + markdown: string; + rawHtml: string; + cleanedHtml: string; + media: MediaAsset[]; + conversionMethod: string; + fallbackReason?: string; +} + +type JsonObject = Record; + +const MIN_CONTENT_LENGTH = 120; +const DEFUDDLE_API_ORIGIN = "https://defuddle.md"; +const LOCAL_FALLBACK_SCORE_DELTA = 120; +const REMOTE_FALLBACK_SCORE_DELTA = 20; +const LOW_QUALITY_MARKERS = [ + /Join The Conversation/i, + /One Community\. Many Voices/i, + /Read our community guidelines/i, + /Create a free account to share your thoughts/i, + /Become a Forbes Member/i, + /Subscribe to trusted journalism/i, + /\bComments\b/i, +]; + +const ARTICLE_TYPES = new Set([ + "Article", + "NewsArticle", + "BlogPosting", + "WebPage", + "ReportageNewsArticle", +]); + +const turndown = new TurndownService({ + headingStyle: "atx", + bulletListMarker: "-", + codeBlockStyle: "fenced", +}) as TurndownService & { + remove(selectors: string[]): void; + addRule( + key: string, + rule: { + filter: string | ((node: Node) => boolean); + replacement: (content: string) => string; + }, + ): void; +}; + +turndown.use(gfm); +turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]); +turndown.addRule("collapseFigure", { + filter: "figure", + replacement(content: string) { + return `\n\n${content.trim()}\n\n`; + }, +}); +turndown.addRule("dropInvisibleAnchors", { + filter(node: Node) { + return ( + node.nodeName === "A" && + !(node as Element).textContent?.trim() && + !(node as Element).querySelector("img, video, picture, source") + ); + }, + replacement() { + return ""; + }, +}); + +function pickString(...values: unknown[]): string | undefined { + for (const value of values) { + if (typeof value !== "string") { + continue; + } + const trimmed = value.trim(); + if (trimmed) { + return trimmed; + } + } + return undefined; +} + +function normalizeMarkdown(markdown: string): string { + return markdown + .replace(/\r\n/g, "\n") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function stripWrappingQuotes(value: string): string { + const trimmed = value.trim(); + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1).trim(); + } + return trimmed; +} + +function stripMarkdownFrontmatter(markdown: string): string { + return markdown.replace(/^\uFEFF?---\n[\s\S]*?\n---(?:\n|$)/, "").trim(); +} + +function cleanMarkdownTitle(value: string): string | undefined { + const cleaned = stripWrappingQuotes( + value + .replace(/\s+#+\s*$/, "") + .replace(/!\[[^\]]*\]\([^)]+\)/g, "") + .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") + .replace(/[*_`~]/g, "") + .trim(), + ); + + return cleaned || undefined; +} + +export function extractTitleFromMarkdownDocument(markdown: string): string | undefined { + const normalized = markdown.replace(/\r\n/g, "\n").trim(); + if (!normalized) { + return undefined; + } + + const frontmatterMatch = normalized.match(/^\uFEFF?---\n([\s\S]*?)\n---(?:\n|$)/); + if (frontmatterMatch) { + for (const line of frontmatterMatch[1].split("\n")) { + const match = line.match(/^title:\s*(.+?)\s*$/i); + if (!match) { + continue; + } + + const title = cleanMarkdownTitle(match[1]); + if (title) { + return title; + } + } + } + + const body = stripMarkdownFrontmatter(normalized); + const headingMatch = body.match(/^#{1,6}\s+(.+)$/m); + if (!headingMatch) { + return undefined; + } + + return cleanMarkdownTitle(headingMatch[1]); +} + +function trimKnownBoilerplate(markdown: string): string { + const normalized = normalizeMarkdown(markdown); + const lines = normalized.split("\n"); + + while (lines.length > 0) { + const lastLine = lines[lines.length - 1]?.trim(); + if (!lastLine) { + lines.pop(); + continue; + } + + if (/^继续滑动看下一个$/.test(lastLine) || /^轻触阅读原文$/.test(lastLine)) { + lines.pop(); + continue; + } + + break; + } + + return normalizeMarkdown(lines.join("\n")); +} + +function buildDefuddleApiUrl(targetUrl: string): string { + return `${DEFUDDLE_API_ORIGIN}/${encodeURIComponent(targetUrl)}`; +} + +async function fetchDefuddleApiMarkdown( + targetUrl: string, +): Promise<{ markdown: string; title?: string }> { + const response = await fetch(buildDefuddleApiUrl(targetUrl), { + headers: { + accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1", + }, + redirect: "follow", + }); + + if (!response.ok) { + throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`); + } + + const rawMarkdown = (await response.text()).replace(/\r\n/g, "\n").trim(); + if (!rawMarkdown) { + throw new Error("defuddle.md returned empty markdown"); + } + + const title = extractTitleFromMarkdownDocument(rawMarkdown); + const markdown = trimKnownBoilerplate(stripMarkdownFrontmatter(rawMarkdown)); + if (!markdown) { + throw new Error("defuddle.md returned empty markdown"); + } + + return { + markdown, + title, + }; +} + +function sanitizeHtmlFragment(html: string): string { + const dom = new JSDOM(`
${html}
`); + const root = dom.window.document.querySelector("#__root"); + if (!root) { + return html; + } + + for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) { + root.querySelectorAll(selector).forEach((element) => element.remove()); + } + + return root.innerHTML; +} + +function extractTextFromHtml(html: string): string { + const dom = new JSDOM(`${html}`); + const { document } = dom.window; + for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) { + document.querySelectorAll(selector).forEach((element) => element.remove()); + } + return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? ""; +} + +function getMetaContent(document: Document, names: string[]): string | undefined { + for (const name of names) { + const element = + document.querySelector(`meta[name="${name}"]`) ?? + document.querySelector(`meta[property="${name}"]`); + const content = element?.getAttribute("content")?.trim(); + if (content) { + return content; + } + } + return undefined; +} + +function normalizeLanguageTag(value: string | null | undefined): string | undefined { + if (!value) { + return undefined; + } + + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + + const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim(); + if (!primary) { + return undefined; + } + + return primary.replace(/_/g, "-"); +} + +function flattenJsonLdItems(data: unknown): JsonObject[] { + if (!data || typeof data !== "object") { + return []; + } + + if (Array.isArray(data)) { + return data.flatMap(flattenJsonLdItems); + } + + const item = data as JsonObject; + if (Array.isArray(item["@graph"])) { + return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems); + } + + return [item]; +} + +function parseJsonLdScripts(document: Document): JsonObject[] { + const results: JsonObject[] = []; + document.querySelectorAll("script[type='application/ld+json']").forEach((script) => { + try { + const data = JSON.parse(script.textContent ?? ""); + results.push(...flattenJsonLdItems(data)); + } catch { + // Ignore malformed json-ld blocks. + } + }); + return results; +} + +function extractAuthorFromJsonLd(authorData: unknown): string | undefined { + if (typeof authorData === "string") { + return authorData.trim() || undefined; + } + + if (!authorData || typeof authorData !== "object") { + return undefined; + } + + if (Array.isArray(authorData)) { + return authorData + .map((author) => extractAuthorFromJsonLd(author)) + .filter((value): value is string => Boolean(value)) + .join(", ") || undefined; + } + + const author = authorData as JsonObject; + return pickString(author.name); +} + +function extractPrimaryJsonLdMeta(document: Document): Partial { + for (const item of parseJsonLdScripts(document)) { + const type = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"]; + if (typeof type !== "string" || !ARTICLE_TYPES.has(type)) { + continue; + } + + return { + title: pickString(item.headline, item.name), + summary: pickString(item.description), + author: extractAuthorFromJsonLd(item.author), + publishedAt: pickString(item.datePublished, item.dateCreated), + coverImage: pickString( + item.image, + (item.image as JsonObject | undefined)?.url, + Array.isArray(item.image) ? item.image[0] : undefined, + ), + }; + } + + return {}; +} + +function extractPageMetadata( + html: string, + url: string, + capturedAt: string, +): HtmlConversionMetadata { + const dom = new JSDOM(html, { url }); + const { document } = dom.window; + const jsonLd = extractPrimaryJsonLdMeta(document); + + return { + url, + canonicalUrl: + document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ?? + getMetaContent(document, ["og:url"]), + siteName: pickString( + getMetaContent(document, ["og:site_name"]), + document.querySelector('meta[name="application-name"]')?.getAttribute("content"), + ), + title: pickString( + getMetaContent(document, ["og:title", "twitter:title"]), + jsonLd.title, + document.querySelector("h1")?.textContent, + document.title, + ), + summary: pickString( + getMetaContent(document, ["description", "og:description", "twitter:description"]), + jsonLd.summary, + ), + author: pickString( + getMetaContent(document, ["author", "article:author", "twitter:creator"]), + jsonLd.author, + ), + publishedAt: pickString( + document.querySelector("time[datetime]")?.getAttribute("datetime"), + getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]), + jsonLd.publishedAt, + ), + coverImage: pickString( + getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]), + jsonLd.coverImage, + ), + language: pickString( + normalizeLanguageTag(document.documentElement.getAttribute("lang")), + normalizeLanguageTag( + pickString( + getMetaContent(document, ["language", "content-language", "og:locale"]), + document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content"), + ), + ), + ), + capturedAt, + }; +} + +function isMarkdownUsable(markdown: string, html: string): boolean { + const normalized = normalizeMarkdown(markdown); + if (!normalized) { + return false; + } + + const htmlTextLength = extractTextFromHtml(html).length; + if (htmlTextLength < MIN_CONTENT_LENGTH) { + return true; + } + + if (normalized.length >= 80) { + return true; + } + + return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2)); +} + +function countMarkerHits(markdown: string, markers: RegExp[]): number { + let hits = 0; + for (const marker of markers) { + if (marker.test(markdown)) { + hits += 1; + } + } + return hits; +} + +function countUsefulParagraphs(markdown: string): number { + const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/); + let count = 0; + + for (const paragraph of paragraphs) { + const trimmed = paragraph.trim(); + if (!trimmed) { + continue; + } + if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) { + continue; + } + if (/^#{1,6}\s+/.test(trimmed)) { + continue; + } + if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) { + continue; + } + count += 1; + } + + return count; +} + +function scoreMarkdownQuality(markdown: string): number { + const normalized = normalizeMarkdown(markdown); + const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length; + const usefulParagraphs = countUsefulParagraphs(normalized); + const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length; + const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS); + return Math.min(wordCount, 4000) + usefulParagraphs * 40 + headingCount * 10 - markerHits * 180; +} + +function shouldCompareWithFallback(markdown: string): boolean { + const normalized = normalizeMarkdown(markdown); + return countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 || countUsefulParagraphs(normalized) < 6; +} + +function hasMeaningfulMarkdownStructure(markdown: string): boolean { + const normalized = normalizeMarkdown(markdown); + if (!normalized) { + return false; + } + + return ( + countUsefulParagraphs(normalized) > 0 || + /^#{1,6}\s+/m.test(normalized) || + /^[-*]\s+/m.test(normalized) || + /^\d+\.\s+/m.test(normalized) || + /!\[[^\]]*\]\([^)]+\)/.test(normalized) + ); +} + +function shouldTryRemoteMarkdownFallback( + markdown: string, + html: string, + options: ConvertHtmlToMarkdownOptions, +): boolean { + if (!options.enableRemoteMarkdownFallback) { + return false; + } + + return !isMarkdownUsable(markdown, html) || shouldCompareWithFallback(markdown); +} + +function shouldPreferRemoteMarkdown( + current: HtmlToMarkdownResult, + remote: HtmlToMarkdownResult, + html: string, +): boolean { + if (!isMarkdownUsable(current.markdown, html)) { + return true; + } + + if (!hasMeaningfulMarkdownStructure(current.markdown) && hasMeaningfulMarkdownStructure(remote.markdown)) { + return true; + } + + return scoreMarkdownQuality(remote.markdown) > scoreMarkdownQuality(current.markdown) + REMOTE_FALLBACK_SCORE_DELTA; +} + +function buildRemoteFallbackReason(current: HtmlToMarkdownResult, html: string): string { + if (!isMarkdownUsable(current.markdown, html)) { + return current.fallbackReason + ? `Used defuddle.md markdown fallback after local extraction failed: ${current.fallbackReason}` + : "Used defuddle.md markdown fallback after local extraction returned empty or incomplete markdown"; + } + + return "defuddle.md produced higher-quality markdown than local extraction"; +} + +async function tryDefuddleConversion( + html: string, + url: string, + baseMetadata: HtmlConversionMetadata, +): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> { + try { + const virtualConsole = new VirtualConsole(); + virtualConsole.on("jsdomError", (error: Error & { type?: string }) => { + if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) { + return; + } + }); + + const dom = new JSDOM(html, { url, virtualConsole }); + const result = await Defuddle(dom, url, { markdown: true }); + const markdown = trimKnownBoilerplate(result.content || ""); + + if (!isMarkdownUsable(markdown, html)) { + return { ok: false, reason: "Defuddle returned empty or incomplete markdown" }; + } + + const metadata: HtmlConversionMetadata = { + ...baseMetadata, + title: pickString(result.title, baseMetadata.title), + summary: pickString(result.description, baseMetadata.summary), + author: pickString(result.author, baseMetadata.author), + publishedAt: pickString(result.published, baseMetadata.publishedAt), + coverImage: pickString(result.image, baseMetadata.coverImage), + language: pickString(result.language, baseMetadata.language), + }; + + return { + ok: true, + result: { + metadata, + markdown, + rawHtml: html, + cleanedHtml: html, + media: collectMediaFromMarkdown(markdown).concat( + metadata.coverImage + ? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }] + : [], + ), + conversionMethod: "defuddle", + }, + }; + } catch (error) { + return { + ok: false, + reason: error instanceof Error ? error.message : String(error), + }; + } +} + +async function tryDefuddleApiConversion( + html: string, + url: string, + baseMetadata: HtmlConversionMetadata, +): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> { + try { + const result = await fetchDefuddleApiMarkdown(url); + const markdown = result.markdown; + + if (!isMarkdownUsable(markdown, html) && scoreMarkdownQuality(markdown) < 80) { + return { ok: false, reason: "defuddle.md returned empty or incomplete markdown" }; + } + + const metadata: HtmlConversionMetadata = { + ...baseMetadata, + title: pickString(result.title, baseMetadata.title), + }; + + return { + ok: true, + result: { + metadata, + markdown, + rawHtml: html, + cleanedHtml: html, + media: collectMediaFromMarkdown(markdown).concat( + metadata.coverImage + ? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }] + : [], + ), + conversionMethod: "defuddle-api", + }, + }; + } catch (error) { + return { + ok: false, + reason: error instanceof Error ? error.message : String(error), + }; + } +} + +function convertHtmlFragmentToMarkdown(html: string): string { + if (!html.trim()) { + return ""; + } + + try { + return turndown.turndown(sanitizeHtmlFragment(html)); + } catch { + return ""; + } +} + +function fallbackPlainText(html: string): string { + return trimKnownBoilerplate(extractTextFromHtml(html)); +} + +function convertWithReadability( + rawHtml: string, + cleanedHtml: string, + url: string, + baseMetadata: HtmlConversionMetadata, +): HtmlToMarkdownResult { + const dom = new JSDOM(cleanedHtml, { url }); + const document = dom.window.document; + const article = new Readability(document).parse(); + + const contentHtml = + article?.content?.trim() ?? + document.querySelector("main")?.innerHTML?.trim() ?? + document.body?.innerHTML?.trim() ?? + ""; + + let markdown = contentHtml ? convertHtmlFragmentToMarkdown(contentHtml) : ""; + if (!markdown) { + markdown = fallbackPlainText(cleanedHtml); + } + + const metadata: HtmlConversionMetadata = { + ...baseMetadata, + title: pickString(article?.title, baseMetadata.title), + summary: pickString(article?.excerpt, baseMetadata.summary), + author: pickString(article?.byline, baseMetadata.author), + }; + + const media = collectMediaFromMarkdown(markdown); + if (metadata.coverImage) { + media.unshift({ + url: metadata.coverImage, + kind: "image", + role: "cover", + }); + } + + return { + metadata, + markdown: trimKnownBoilerplate(markdown), + rawHtml, + cleanedHtml, + media, + conversionMethod: article?.content ? "legacy:readability" : "legacy:body", + }; +} + +export async function convertHtmlToMarkdown( + html: string, + url: string, + options: ConvertHtmlToMarkdownOptions = {}, +): Promise { + const capturedAt = new Date().toISOString(); + const baseMetadata = extractPageMetadata(html, url, capturedAt); + + let cleanedHtml = html; + try { + cleanedHtml = cleanHtml(html, url, { + removeBase64Images: !options.preserveBase64Images, + }); + } catch { + cleanedHtml = html; + } + + let selectedResult: HtmlToMarkdownResult; + const defuddleResult = await tryDefuddleConversion(cleanedHtml, url, baseMetadata); + if (defuddleResult.ok) { + if (shouldCompareWithFallback(defuddleResult.result.markdown)) { + const fallbackResult = convertWithReadability(html, cleanedHtml, url, baseMetadata); + if ( + scoreMarkdownQuality(fallbackResult.markdown) > + scoreMarkdownQuality(defuddleResult.result.markdown) + LOCAL_FALLBACK_SCORE_DELTA + ) { + selectedResult = { + ...fallbackResult, + fallbackReason: "Readability/Turndown produced higher-quality markdown than Defuddle", + }; + } else { + selectedResult = { + ...defuddleResult.result, + rawHtml: html, + cleanedHtml, + }; + } + } else { + selectedResult = { + ...defuddleResult.result, + rawHtml: html, + cleanedHtml, + }; + } + } else { + selectedResult = { + ...convertWithReadability(html, cleanedHtml, url, baseMetadata), + fallbackReason: defuddleResult.reason, + }; + } + + if (!shouldTryRemoteMarkdownFallback(selectedResult.markdown, cleanedHtml, options)) { + return selectedResult; + } + + const remoteDefuddleResult = await tryDefuddleApiConversion(cleanedHtml, url, baseMetadata); + if (!remoteDefuddleResult.ok || !shouldPreferRemoteMarkdown(selectedResult, remoteDefuddleResult.result, cleanedHtml)) { + return selectedResult; + } + + return { + ...remoteDefuddleResult.result, + rawHtml: html, + cleanedHtml, + fallbackReason: buildRemoteFallbackReason(selectedResult, cleanedHtml), + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts new file mode 100644 index 0000000..e843583 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts @@ -0,0 +1,169 @@ +import TurndownService from "turndown"; +import { gfm } from "turndown-plugin-gfm"; +import { normalizeMarkdownMediaLinks } from "../media/markdown-media"; +import type { ContentBlock, ExtractedDocument } from "./document"; + +const turndownService = new TurndownService({ + codeBlockStyle: "fenced", + headingStyle: "atx", + bulletListMarker: "-", +}); + +turndownService.use(gfm); + +function renderBlock(block: ContentBlock): string { + switch (block.type) { + case "paragraph": + return block.text.trim(); + case "heading": + return `${"#".repeat(Math.min(Math.max(block.depth, 1), 6))} ${block.text.trim()}`; + case "list": + return block.items + .map((item, index) => (block.ordered ? `${index + 1}. ${item.trim()}` : `- ${item.trim()}`)) + .join("\n"); + case "quote": + return block.text + .split("\n") + .map((line) => `> ${line}`) + .join("\n"); + case "code": + return `\`\`\`${block.language ?? ""}\n${block.code.trimEnd()}\n\`\`\``; + case "image": + return `![${block.alt ?? ""}](${block.url})`; + case "html": + return turndownService.turndown(block.html).trim(); + case "markdown": + return block.markdown.trim(); + } +} + +function isDefinedValue(value: unknown): boolean { + return value !== undefined && value !== null && value !== ""; +} + +function renderFrontmatterValue(value: unknown): string { + if (typeof value === "string") { + if (value.includes("\n")) { + return `|-\n${value + .replace(/\r\n/g, "\n") + .split("\n") + .map((line) => ` ${line}`) + .join("\n")}`; + } + return JSON.stringify(value); + } + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + return JSON.stringify(value); +} + +function renderFrontmatter(document: ExtractedDocument): string { + const fields = new Map(); + const preferredOrder = [ + "title", + "url", + "requestedUrl", + "author", + "authorName", + "authorUsername", + "authorUrl", + "coverImage", + "siteName", + "publishedAt", + "summary", + "adapter", + ]; + + fields.set("title", document.title); + fields.set("url", document.canonicalUrl ?? document.url); + fields.set("requestedUrl", document.requestedUrl ?? document.url); + fields.set("author", document.author); + fields.set("siteName", document.siteName); + fields.set("publishedAt", document.publishedAt); + fields.set("summary", document.summary); + fields.set("adapter", document.adapter); + + for (const [key, value] of Object.entries(document.metadata ?? {})) { + if (!fields.has(key)) { + fields.set(key, value); + } + } + + const orderedKeys = [ + ...preferredOrder.filter((key) => fields.has(key)), + ...Array.from(fields.keys()).filter((key) => !preferredOrder.includes(key)).sort(), + ]; + + const lines = orderedKeys + .map((key) => [key, fields.get(key)] as const) + .filter(([, value]) => isDefinedValue(value)) + .map(([key, value]) => `${key}: ${renderFrontmatterValue(value)}`); + + if (lines.length === 0) { + return ""; + } + + return `---\n${lines.join("\n")}\n---`; +} + +function cleanMarkdown(markdown: string): string { + return normalizeMarkdownMediaLinks(markdown.replace(/\n{3,}/g, "\n\n").trim()); +} + +function normalizeComparableTitle(value: string): string { + return value + .trim() + .toLowerCase() + .replace(/^>\s*/, "") + .replace(/^#+\s+/, "") + .replace(/(?:\.{3}|…)\s*$/, ""); +} + +function bodyStartsWithTitle(body: string, title: string): boolean { + const firstMeaningfulLine = body + .replace(/\r\n/g, "\n") + .split("\n") + .map((line) => line.trim()) + .find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line)); + + if (!firstMeaningfulLine) { + return false; + } + + const comparableTitle = normalizeComparableTitle(title); + const comparableFirstLine = normalizeComparableTitle(firstMeaningfulLine); + if (!comparableTitle || !comparableFirstLine) { + return false; + } + + return ( + comparableFirstLine === comparableTitle || + comparableFirstLine.startsWith(comparableTitle) || + comparableTitle.startsWith(comparableFirstLine) + ); +} + +export function renderMarkdown(document: ExtractedDocument): string { + const sections: string[] = []; + const frontmatter = renderFrontmatter(document); + + if (frontmatter) { + sections.push(frontmatter); + } + + const body = document.content + .map((block) => renderBlock(block)) + .filter(Boolean) + .join("\n\n"); + + if (document.title && !bodyStartsWithTitle(body, document.title)) { + sections.push(`# ${document.title}`); + } + + if (body) { + sections.push(body); + } + + return cleanMarkdown(sections.join("\n\n")); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts new file mode 100644 index 0000000..3d11e0e --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts @@ -0,0 +1,161 @@ +import path from "node:path"; +import { mkdir, writeFile } from "node:fs/promises"; +import { + buildFileName, + isDataUri, + normalizeContentType, + normalizeMediaUrl, + resolveExtensionFromContentType, + resolveExtensionFromUrl, + resolveMediaKind, + resolveOutputExtension, + toPosixPath, +} from "./media-utils"; +import type { MediaAsset, MediaDownloadRequest, MediaDownloadResult, MediaKind } from "./types"; + +const DOWNLOAD_USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; + +function parseBase64DataUri(rawUrl: string): { contentType: string; bytes: Buffer } | null { + const match = rawUrl.match(/^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/i); + if (!match?.[1] || !match[2]) { + return null; + } + + const contentType = normalizeContentType(match[1]); + if (!contentType) { + return null; + } + + try { + const bytes = Buffer.from(match[2].replace(/\s+/g, ""), "base64"); + if (bytes.length === 0) { + return null; + } + return { contentType, bytes }; + } catch { + return null; + } +} + +function dedupeMedia(media: MediaAsset[]): MediaAsset[] { + const deduped: MediaAsset[] = []; + const seen = new Set(); + for (const item of media) { + const normalizedUrl = normalizeMediaUrl(item.url); + if (!normalizedUrl || seen.has(normalizedUrl)) { + continue; + } + seen.add(normalizedUrl); + deduped.push({ + ...item, + url: normalizedUrl, + }); + } + return deduped; +} + +function toRelativePath(fromDir: string, absoluteTarget: string): string { + const relative = path.relative(fromDir, absoluteTarget) || path.basename(absoluteTarget); + return toPosixPath(relative); +} + +export async function downloadMediaAssets( + request: MediaDownloadRequest, +): Promise { + const dedupedMedia = dedupeMedia(request.media); + const absoluteOutputPath = path.resolve(request.outputPath); + const markdownDir = path.dirname(absoluteOutputPath); + const baseDir = request.mediaDir ? path.resolve(request.mediaDir) : markdownDir; + const replacements: MediaDownloadResult["replacements"] = []; + + let downloadedImages = 0; + let downloadedVideos = 0; + + for (const asset of dedupedMedia) { + try { + let sourceUrl = normalizeMediaUrl(asset.url); + let contentType = ""; + let extension: string | undefined; + let kind: MediaKind | undefined; + let bytes: Buffer | null = null; + + if (isDataUri(asset.url)) { + const parsed = parseBase64DataUri(asset.url); + if (!parsed) { + request.log.warn(`Skipping unsupported embedded media: ${asset.url.slice(0, 32)}...`); + continue; + } + + contentType = parsed.contentType; + extension = + resolveExtensionFromContentType(contentType) ?? + resolveExtensionFromUrl(asset.fileNameHint ?? ""); + kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind); + bytes = parsed.bytes; + } else { + const response = await fetch(sourceUrl, { + method: "GET", + redirect: "follow", + headers: { + "user-agent": DOWNLOAD_USER_AGENT, + ...(asset.headers ?? {}), + }, + }); + + if (!response.ok) { + request.log.warn(`Skipping media (${response.status}): ${asset.url}`); + continue; + } + + sourceUrl = normalizeMediaUrl(response.url || sourceUrl); + contentType = normalizeContentType(response.headers.get("content-type")); + extension = + resolveExtensionFromUrl(sourceUrl) ?? + resolveExtensionFromUrl(asset.url) ?? + resolveExtensionFromUrl(asset.fileNameHint ?? ""); + kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind); + bytes = Buffer.from(await response.arrayBuffer()); + } + + if (!kind || !bytes) { + request.log.debug(`Skipping media with unresolved kind: ${asset.url}`); + continue; + } + + const outputExtension = resolveOutputExtension(contentType, extension, kind); + const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1; + const dirName = kind === "image" ? "imgs" : "videos"; + const targetDir = path.join(baseDir, dirName); + await mkdir(targetDir, { recursive: true }); + + const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension, asset.fileNameHint); + const absolutePath = path.join(targetDir, fileName); + await writeFile(absolutePath, bytes); + + replacements.push({ + url: asset.url, + localPath: toRelativePath(markdownDir, absolutePath), + absolutePath, + kind, + }); + + if (kind === "image") { + downloadedImages = nextIndex; + } else { + downloadedVideos = nextIndex; + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + request.log.warn(`Failed to download media ${asset.url}: ${message}`); + } + } + + return { + replacements, + downloadedImages, + downloadedVideos, + imageDir: downloadedImages > 0 ? path.join(baseDir, "imgs") : null, + videoDir: downloadedVideos > 0 ? path.join(baseDir, "videos") : null, + }; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts new file mode 100644 index 0000000..a930671 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts @@ -0,0 +1,458 @@ +import remarkGfm from "remark-gfm"; +import remarkParse from "remark-parse"; +import { unified } from "unified"; +import type { ContentBlock, ExtractedDocument } from "../extract/document"; +import { + isDataUri, + normalizeContentType, + normalizeMediaUrl, + resolveExtensionFromContentType, + resolveExtensionFromUrl, + resolveKindFromExtension, +} from "./media-utils"; +import type { MediaAsset, MediaReplacement } from "./types"; + +const MARKDOWN_LINK_RE = + /(!?\[[^\]\n]*\])\((<)?((?:https?:\/\/[^)\s>]+)|(?:data:[^)>\s]+))(>)?\)/g; +const FRONTMATTER_COVER_RE = /^(coverImage:\s*")((?:https?:\/\/[^"]+)|(?:data:[^"]+))(")/m; +const RAW_URL_RE = /(?:https?:\/\/[^\s<>"')\]]+|data:[^\s<>"')\]]+)/g; + +interface MarkdownAstNode { + type: string; + url?: string | null; + alt?: string | null; + title?: string | null; + value?: string | null; + children?: MarkdownAstNode[]; + position?: { + start?: { offset?: number | null }; + end?: { offset?: number | null }; + }; +} + +interface MarkdownReplacementRange { + start: number; + end: number; + value: string; +} + +function inferMediaKindFromLabel(label: string, rawUrl: string): "image" | "video" | undefined { + if (label.startsWith("![")) { + return "image"; + } + + const normalizedLabel = label.replace(/[!\[\]]/g, "").trim().toLowerCase(); + if (/\b(video|animated[_ -]?gif|gif)\b/.test(normalizedLabel)) { + return "video"; + } + + if (isDataUri(rawUrl)) { + const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";"))); + return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined; + } + + return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl)); +} + +function inferMediaKindFromRawUrl(rawUrl: string): "image" | "video" | undefined { + if (isDataUri(rawUrl)) { + const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";"))); + return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined; + } + + return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl)); +} + +function pushMedia(assets: MediaAsset[], seen: Set, media: MediaAsset): void { + const normalizedUrl = normalizeMediaUrl(media.url); + if (!normalizedUrl || seen.has(normalizedUrl)) { + return; + } + seen.add(normalizedUrl); + assets.push({ + ...media, + url: normalizedUrl, + }); +} + +function getNodeOffsets(node: MarkdownAstNode): { start: number; end: number } | null { + const start = node.position?.start?.offset; + const end = node.position?.end?.offset; + if (typeof start !== "number" || typeof end !== "number" || start < 0 || end < start) { + return null; + } + return { start, end }; +} + +function escapeMarkdownLabel(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/\[/g, "\\[").replace(/\]/g, "\\]"); +} + +function escapeMarkdownTitle(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); +} + +function formatMarkdownDestination(url: string): string { + return /[\s()<>]/.test(url) ? `<${url}>` : url; +} + +function serializeImageNode(node: MarkdownAstNode): string { + const rawUrl = node.url ?? ""; + const normalizedUrl = normalizeMediaUrl(rawUrl); + const alt = escapeMarkdownLabel(node.alt ?? ""); + const title = node.title ? ` "${escapeMarkdownTitle(node.title)}"` : ""; + return `![${alt}](${formatMarkdownDestination(normalizedUrl)}${title})`; +} + +function serializeLinkedImageNode(linkNode: MarkdownAstNode, imageNode: MarkdownAstNode): string { + const imageMarkdown = serializeImageNode(imageNode); + const imageUrl = normalizeMediaUrl(imageNode.url ?? ""); + const linkUrl = normalizeMediaUrl(linkNode.url ?? ""); + + if (!linkUrl || linkUrl === imageUrl) { + return imageMarkdown; + } + + const title = linkNode.title ? ` "${escapeMarkdownTitle(linkNode.title)}"` : ""; + return `[${imageMarkdown}](${formatMarkdownDestination(linkUrl)}${title})`; +} + +function isParagraphWithSingleText(node: MarkdownAstNode | undefined, expectedValue: string): boolean { + if (node?.type !== "paragraph" || node.children?.length !== 1) { + return false; + } + + const child = node.children[0]; + return child?.type === "text" && child.value?.trim() === expectedValue; +} + +function getSingleImageFromParagraph(node: MarkdownAstNode | undefined): MarkdownAstNode | null { + if (node?.type !== "paragraph" || node.children?.length !== 1) { + return null; + } + + return node.children[0]?.type === "image" ? node.children[0] : null; +} + +function extractBrokenLinkedImageDestination(node: MarkdownAstNode | undefined): string | null { + if (node?.type !== "paragraph") { + return null; + } + + const children = node.children ?? []; + if (children.length !== 3) { + return null; + } + + const [prefix, linkNode, suffix] = children; + if (prefix?.type !== "text" || prefix.value?.trim() !== "](") { + return null; + } + if (linkNode?.type !== "link" || !linkNode.url) { + return null; + } + if (suffix?.type !== "text" || suffix.value?.trim() !== ")") { + return null; + } + + return linkNode.url; +} + +function collectLinkedImageReplacements( + node: MarkdownAstNode, + replacements: MarkdownReplacementRange[], +): void { + const children = node.children ?? []; + + if (node.type === "link" && children.length === 1 && children[0]?.type === "image") { + const offsets = getNodeOffsets(node); + if (offsets) { + replacements.push({ + start: offsets.start, + end: offsets.end, + value: serializeLinkedImageNode(node, children[0]), + }); + } + return; + } + + for (const child of children) { + collectLinkedImageReplacements(child, replacements); + } +} + +function collectBrokenLinkedImageReplacements( + node: MarkdownAstNode, + replacements: MarkdownReplacementRange[], +): void { + const children = node.children ?? []; + for (let index = 0; index <= children.length - 3; index += 1) { + const openParagraph = children[index]; + const imageParagraph = children[index + 1]; + const closeParagraph = children[index + 2]; + + if (!isParagraphWithSingleText(openParagraph, "[")) { + continue; + } + + const imageNode = getSingleImageFromParagraph(imageParagraph); + if (!imageNode) { + continue; + } + + const linkUrl = extractBrokenLinkedImageDestination(closeParagraph); + if (!linkUrl) { + continue; + } + + const start = openParagraph.position?.start?.offset; + const end = closeParagraph.position?.end?.offset; + if (typeof start !== "number" || typeof end !== "number" || end < start) { + continue; + } + + replacements.push({ + start, + end, + value: serializeLinkedImageNode({ type: "link", url: linkUrl }, imageNode), + }); + + index += 2; + } + + for (const child of children) { + collectBrokenLinkedImageReplacements(child, replacements); + } +} + +function applyReplacements(source: string, replacements: MarkdownReplacementRange[]): string { + if (replacements.length === 0) { + return source; + } + + let result = source; + const sorted = [...replacements].sort((left, right) => right.start - left.start); + for (const replacement of sorted) { + result = `${result.slice(0, replacement.start)}${replacement.value}${result.slice(replacement.end)}`; + } + return result; +} + +function normalizeLinkedImageMarkdown(markdown: string): string { + let tree: MarkdownAstNode; + try { + tree = unified().use(remarkParse).use(remarkGfm).parse(markdown) as MarkdownAstNode; + } catch { + return markdown; + } + + const replacements: MarkdownReplacementRange[] = []; + collectLinkedImageReplacements(tree, replacements); + collectBrokenLinkedImageReplacements(tree, replacements); + return applyReplacements(markdown, replacements); +} + +export function normalizeMarkdownMediaLinks(markdown: string): string { + MARKDOWN_LINK_RE.lastIndex = 0; + let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, openAngle, rawUrl, closeAngle) => { + const normalizedUrl = normalizeMediaUrl(rawUrl); + if (normalizedUrl === rawUrl) { + return full; + } + return `${label}(${openAngle ?? ""}${normalizedUrl}${closeAngle ?? ""})`; + }); + + result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => { + const normalizedUrl = normalizeMediaUrl(rawUrl); + if (normalizedUrl === rawUrl) { + return full; + } + return `${prefix}${normalizedUrl}${suffix}`; + }); + + RAW_URL_RE.lastIndex = 0; + result = result.replace(RAW_URL_RE, (rawUrl) => normalizeMediaUrl(rawUrl)); + return normalizeLinkedImageMarkdown(result); +} + +export function collectMediaFromText( + text: string, + options: { + role?: MediaAsset["role"]; + defaultKind?: MediaAsset["kind"]; + seen?: Set; + into?: MediaAsset[]; + } = {}, +): MediaAsset[] { + const assets = options.into ?? []; + const seen = options.seen ?? new Set(); + + MARKDOWN_LINK_RE.lastIndex = 0; + let linkMatch: RegExpExecArray | null; + while ((linkMatch = MARKDOWN_LINK_RE.exec(text))) { + const label = linkMatch[1] ?? ""; + const rawUrl = linkMatch[3] ?? ""; + const kind = inferMediaKindFromLabel(label, rawUrl) ?? options.defaultKind; + if (!kind) { + continue; + } + pushMedia(assets, seen, { + url: rawUrl, + kind, + role: options.role ?? "inline", + }); + } + + RAW_URL_RE.lastIndex = 0; + let rawMatch: RegExpExecArray | null; + while ((rawMatch = RAW_URL_RE.exec(text))) { + const rawUrl = rawMatch[0] ?? ""; + const kind = inferMediaKindFromRawUrl(rawUrl) ?? options.defaultKind; + if (!kind) { + continue; + } + pushMedia(assets, seen, { + url: rawUrl, + kind, + role: options.role ?? "inline", + }); + } + + return assets; +} + +function collectMediaFromBlock( + block: ContentBlock, + assets: MediaAsset[], + seen: Set, +): void { + switch (block.type) { + case "image": + pushMedia(assets, seen, { + url: block.url, + kind: "image", + role: "inline", + alt: block.alt, + }); + return; + case "html": + case "markdown": + collectMediaFromText(block.type === "html" ? block.html : block.markdown, { + role: "inline", + seen, + into: assets, + }); + return; + case "paragraph": + case "quote": + collectMediaFromText(block.text, { + role: "inline", + seen, + into: assets, + }); + return; + case "list": + for (const item of block.items) { + collectMediaFromText(item, { + role: "attachment", + seen, + into: assets, + }); + } + return; + case "heading": + case "code": + return; + } +} + +export function collectMediaFromDocument(document: ExtractedDocument): MediaAsset[] { + const assets: MediaAsset[] = []; + const seen = new Set(); + const coverImage = + typeof document.metadata?.coverImage === "string" ? document.metadata.coverImage : undefined; + + if (coverImage) { + pushMedia(assets, seen, { + url: coverImage, + kind: "image", + role: "cover", + }); + } + + for (const block of document.content) { + collectMediaFromBlock(block, assets, seen); + } + + return assets; +} + +export function collectMediaFromMarkdown(markdown: string): MediaAsset[] { + const assets: MediaAsset[] = []; + const seen = new Set(); + const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/); + if (fmMatch) { + const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE); + if (coverMatch?.[2]) { + pushMedia(assets, seen, { + url: coverMatch[2], + kind: "image", + role: "cover", + }); + } + } + + collectMediaFromText(markdown, { seen, into: assets }); + return assets; +} + +export function rewriteMarkdownMediaLinks( + markdown: string, + replacements: MediaReplacement[], +): string { + if (replacements.length === 0) { + return markdown; + } + + const replacementMap = new Map(); + for (const item of replacements) { + replacementMap.set(item.url, item.localPath); + replacementMap.set(normalizeMediaUrl(item.url), item.localPath); + } + + MARKDOWN_LINK_RE.lastIndex = 0; + let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, _openAngle, rawUrl) => { + const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl)); + if (!replacement) { + return full; + } + return `${label}(${replacement})`; + }); + + result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => { + const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl)); + if (!replacement) { + return full; + } + return `${prefix}${replacement}${suffix}`; + }); + + for (const { url, localPath } of replacements) { + result = result.split(url).join(localPath); + const normalizedUrl = normalizeMediaUrl(url); + if (normalizedUrl !== url) { + result = result.split(normalizedUrl).join(localPath); + } + } + + return result; +} + +export function resolveDataUriExtension(rawUrl: string): string | undefined { + if (!isDataUri(rawUrl)) { + return undefined; + } + const separatorIndex = rawUrl.indexOf(";"); + const contentType = normalizeContentType(rawUrl.slice(5, separatorIndex === -1 ? undefined : separatorIndex)); + return resolveExtensionFromContentType(contentType); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts new file mode 100644 index 0000000..30a7447 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts @@ -0,0 +1,261 @@ +import path from "node:path"; +import type { MediaKind } from "./types"; + +const IMAGE_EXTENSIONS = new Set([ + "jpg", + "jpeg", + "png", + "webp", + "gif", + "bmp", + "avif", + "heic", + "heif", + "svg", +]); + +const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]); + +const MIME_EXTENSION_MAP: Record = { + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/bmp": "bmp", + "image/avif": "avif", + "image/heic": "heic", + "image/heif": "heif", + "image/svg+xml": "svg", + "video/mp4": "mp4", + "video/webm": "webm", + "video/quicktime": "mov", + "video/x-m4v": "m4v", +}; + +export function normalizeContentType(raw: string | null): string { + return raw?.split(";")[0]?.trim().toLowerCase() ?? ""; +} + +export function normalizeExtension(raw: string | undefined | null): string | undefined { + if (!raw) { + return undefined; + } + const trimmed = raw.replace(/^\./, "").trim().toLowerCase(); + if (!trimmed) { + return undefined; + } + if (trimmed === "jpeg" || trimmed === "jpg") { + return "jpg"; + } + return trimmed; +} + +export function resolveExtensionFromUrl(rawUrl: string): string | undefined { + try { + const parsed = new URL(rawUrl); + const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname)); + if (extFromPath) { + return extFromPath; + } + const extFromFormat = normalizeExtension(parsed.searchParams.get("format")); + if (extFromFormat) { + return extFromFormat; + } + } catch { + return undefined; + } + return undefined; +} + +export function resolveExtensionFromContentType(contentType: string): string | undefined { + return normalizeExtension(MIME_EXTENSION_MAP[contentType]); +} + +export function resolveKindFromContentType(contentType: string): MediaKind | undefined { + if (!contentType) { + return undefined; + } + if (contentType.startsWith("image/")) { + return "image"; + } + if (contentType.startsWith("video/")) { + return "video"; + } + return undefined; +} + +export function resolveKindFromExtension(extension: string | undefined): MediaKind | undefined { + if (!extension) { + return undefined; + } + if (IMAGE_EXTENSIONS.has(extension)) { + return "image"; + } + if (VIDEO_EXTENSIONS.has(extension)) { + return "video"; + } + return undefined; +} + +export function resolveMediaKind( + rawUrl: string, + contentType: string, + extension: string | undefined, + hint?: MediaKind, +): MediaKind | undefined { + const kindFromType = resolveKindFromContentType(contentType); + if (kindFromType) { + return kindFromType; + } + + const kindFromExtension = resolveKindFromExtension(extension); + if (kindFromExtension) { + return kindFromExtension; + } + + if (contentType && contentType !== "application/octet-stream") { + return undefined; + } + + if (hint) { + return hint; + } + + if (rawUrl.startsWith("data:image/")) { + return "image"; + } + + if (rawUrl.startsWith("data:video/")) { + return "video"; + } + + return undefined; +} + +export function resolveOutputExtension( + contentType: string, + extension: string | undefined, + kind: MediaKind, +): string { + const fromMime = resolveExtensionFromContentType(contentType); + if (fromMime) { + return fromMime; + } + const normalized = normalizeExtension(extension); + if (normalized) { + return normalized; + } + return kind === "video" ? "mp4" : "jpg"; +} + +export function isDataUri(value: string): boolean { + return value.startsWith("data:"); +} + +export function safeDecodeURIComponent(value: string): string { + try { + return decodeURIComponent(value); + } catch { + return value; + } +} + +function extractEmbeddedUrl(value: string): string | undefined { + const encodedMatch = value.match(/https?%3A%2F%2F.+$/i)?.[0]; + if (encodedMatch) { + const decoded = safeDecodeURIComponent(encodedMatch); + try { + return new URL(decoded).href; + } catch { + return undefined; + } + } + + const literalMatch = value.match(/https?:\/\/.+$/i)?.[0]; + if (!literalMatch) { + return undefined; + } + + try { + return new URL(literalMatch).href; + } catch { + return undefined; + } +} + +export function normalizeMediaUrl(rawUrl: string): string { + if (isDataUri(rawUrl)) { + return rawUrl; + } + + try { + const parsed = new URL(rawUrl); + const hostname = parsed.hostname.toLowerCase(); + + if (hostname === "substackcdn.com" || hostname.endsWith(".substackcdn.com")) { + const embeddedUrl = extractEmbeddedUrl(`${parsed.pathname}${parsed.search}`); + if (embeddedUrl) { + return embeddedUrl; + } + } + + return parsed.href; + } catch { + return rawUrl; + } +} + +export function sanitizeFileSegment(input: string): string { + return input + .replace(/[^a-zA-Z0-9_-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-_]+|[-_]+$/g, "") + .slice(0, 48); +} + +export function resolveFileStem(rawUrl: string, extension: string, fileNameHint?: string): string { + const hintBase = fileNameHint?.trim(); + if (hintBase) { + const parsed = path.posix.parse(hintBase); + const stem = parsed.name || parsed.base; + return sanitizeFileSegment(stem); + } + + if (isDataUri(rawUrl)) { + return ""; + } + + try { + const parsed = new URL(rawUrl); + const base = path.posix.basename(parsed.pathname); + if (!base) { + return ""; + } + const decodedBase = safeDecodeURIComponent(base); + const normalizedExtension = normalizeExtension(extension); + const stripExtension = normalizedExtension ? new RegExp(`\\.${normalizedExtension}$`, "i") : null; + const rawStem = stripExtension ? decodedBase.replace(stripExtension, "") : decodedBase; + return sanitizeFileSegment(rawStem); + } catch { + return ""; + } +} + +export function buildFileName( + kind: MediaKind, + index: number, + sourceUrl: string, + extension: string, + fileNameHint?: string, +): string { + const stem = resolveFileStem(sourceUrl, extension, fileNameHint); + const prefix = kind === "image" ? "img" : "video"; + const serial = String(index).padStart(3, "0"); + const suffix = stem ? `-${stem}` : ""; + return `${prefix}-${serial}${suffix}.${extension}`; +} + +export function toPosixPath(value: string): string { + return value.split(path.sep).join(path.posix.sep); +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/types.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/types.ts new file mode 100644 index 0000000..00cae8f --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/media/types.ts @@ -0,0 +1,34 @@ +import type { Logger } from "../utils/logger"; + +export type MediaKind = "image" | "video"; + +export interface MediaAsset { + url: string; + kind?: MediaKind; + role?: "cover" | "inline" | "attachment"; + alt?: string; + fileNameHint?: string; + headers?: Record; +} + +export interface MediaReplacement { + url: string; + localPath: string; + absolutePath: string; + kind: MediaKind; +} + +export interface MediaDownloadRequest { + media: MediaAsset[]; + outputPath: string; + mediaDir?: string; + log: Logger; +} + +export interface MediaDownloadResult { + replacements: MediaReplacement[]; + downloadedImages: number; + downloadedVideos: number; + imageDir: string | null; + videoDir: string | null; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts b/skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts new file mode 100644 index 0000000..ec23b3a --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts @@ -0,0 +1,31 @@ +declare module "defuddle/node" { + export interface DefuddleResponse { + content?: string; + title?: string; + description?: string; + author?: string; + published?: string; + image?: string; + language?: string; + } + + export interface DefuddleOptions { + markdown?: boolean; + } + + export function Defuddle( + input: + | Document + | string + | { + window: { + document: Document; + location: { + href: string; + }; + }; + }, + url?: string, + options?: DefuddleOptions, + ): Promise; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts b/skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts new file mode 100644 index 0000000..c29bbe9 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts @@ -0,0 +1,17 @@ +declare module "turndown" { + export interface TurndownOptions { + codeBlockStyle?: "indented" | "fenced"; + headingStyle?: "setext" | "atx"; + bulletListMarker?: "-" | "*" | "+"; + } + + export default class TurndownService { + constructor(options?: TurndownOptions); + use(plugin: unknown): void; + turndown(input: string): string; + } +} + +declare module "turndown-plugin-gfm" { + export const gfm: unknown; +} diff --git a/skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts b/skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts new file mode 100644 index 0000000..cde9e19 --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts @@ -0,0 +1,30 @@ +export interface Logger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; + debug(message: string): void; +} + +export function createLogger(debugEnabled = false): Logger { + const print = (level: string, message: string): void => { + console.error(`[${level}] ${message}`); + }; + + return { + info(message: string) { + print("info", message); + }, + warn(message: string) { + print("warn", message); + }, + error(message: string) { + print("error", message); + }, + debug(message: string) { + if (debugEnabled) { + print("debug", message); + } + }, + }; +} + diff --git a/skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts b/skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts new file mode 100644 index 0000000..578e97a --- /dev/null +++ b/skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts @@ -0,0 +1,12 @@ +export function normalizeUrl(input: string): URL { + try { + return new URL(input); + } catch { + throw new Error(`Invalid URL: ${input}`); + } +} + +export function sanitizeFilename(input: string): string { + return input.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "document"; +} + diff --git a/skills/baoyu-url-to-markdown/scripts/package.json b/skills/baoyu-url-to-markdown/scripts/package.json index 183f70a..850d688 100644 --- a/skills/baoyu-url-to-markdown/scripts/package.json +++ b/skills/baoyu-url-to-markdown/scripts/package.json @@ -2,7 +2,25 @@ "name": "baoyu-url-to-markdown-scripts", "private": true, "type": "module", + "bin": { + "baoyu-fetch": "./baoyu-fetch" + }, + "scripts": { + "reader": "bun ./lib/cli.ts" + }, "dependencies": { - "baoyu-fetch": "^0.1.2" + "@mozilla/readability": "^0.6.0", + "chrome-launcher": "^1.2.1", + "defuddle": "^0.17.0", + "jsdom": "^29.0.2", + "remark-gfm": "^4.0.1", + "remark-parse": "^11.0.0", + "turndown": "^7.2.0", + "turndown-plugin-gfm": "^1.0.2", + "unified": "^11.0.5", + "ws": "^8.18.3" + }, + "overrides": { + "@xmldom/xmldom": "0.8.13" } }