feat(baoyu-url-to-markdown): add browser fallback strategy, content cleaner, and data URI support

- Browser strategy: headless first with automatic retry in visible Chrome on failure - New --browser auto|headless|headed flag with --headless/--headed shortcuts - Content cleaner module for HTML preprocessing (remove ads, base64 images, scripts) - Media localizer now handles base64 data URIs alongside remote URLs - Capture finalUrl from browser to track redirects for output path - Agent quality gate documentation for post-capture validation - Upgrade defuddle ^0.12.0 → ^0.14.0 - Add unit tests for content-cleaner, html-to-markdown, legacy-converter, media-localizer
2026-07-21 01:39:48 +08:00 · 2026-03-24 22:39:17 -05:00
parent 40f9f05c22
commit e99ce744cd
12 changed files with 909 additions and 67 deletions
@@ -0,0 +1,55 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { cleanContent } from "./content-cleaner.js";
+
+const SAMPLE_HTML = `<!doctype html>
+<html>
+  <head>
+    <title>Example Story</title>
+    <style>.cookie-banner { position: fixed; }</style>
+    <script>window.__noise = true;</script>
+  </head>
+  <body>
+    <!-- comment that should be removed -->
+    <header>
+      <nav>
+        <a href="/home">Home</a>
+        <a href="/topics">Topics</a>
+      </nav>
+    </header>
+    <div class="cookie-banner">Accept cookies</div>
+    <aside>Sidebar links</aside>
+    <main>
+      <article class="content">
+        <h1>Actual Story Title</h1>
+        <p>
+          This is the first paragraph of the real story body, and it is intentionally long enough
+          to survive the cleaner's main-content heuristics without being mistaken for navigation.
+        </p>
+        <p>
+          This is the second paragraph with more useful detail, a
+          <a href="/read-more">supporting link</a>, and a normal image.
+        </p>
+        <img src="/images/cover.jpg" alt="Cover">
+        <img src="data:image/png;base64,AAAA" alt="Inline data">
+      </article>
+    </main>
+    <footer>Footer boilerplate</footer>
+  </body>
+</html>`;
+
+test("cleanContent keeps the article body and removes obvious boilerplate", () => {
+  const cleaned = cleanContent(SAMPLE_HTML, "https://example.com/posts/story");
+
+  assert.match(cleaned, /Actual Story Title/);
+  assert.match(cleaned, /https:\/\/example\.com\/read-more/);
+  assert.match(cleaned, /https:\/\/example\.com\/images\/cover\.jpg/);
+
+  assert.doesNotMatch(cleaned, /Accept cookies/);
+  assert.doesNotMatch(cleaned, /Sidebar links/);
+  assert.doesNotMatch(cleaned, /Footer boilerplate/);
+  assert.doesNotMatch(cleaned, /window\.__noise/);
+  assert.doesNotMatch(cleaned, /comment that should be removed/);
+  assert.doesNotMatch(cleaned, /data:image\/png;base64/);
+});