seahi/baoyu-skills
1
0
Fork 0
mirror of https://github.com/JimLiu/baoyu-skills.git synced 2026-07-21 09:49:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(baoyu-url-to-markdown): add browser fallback strategy, content cleaner, and data URI support

Browse Source 
- Browser strategy: headless first with automatic retry in visible Chrome on failure
- New --browser auto|headless|headed flag with --headless/--headed shortcuts
- Content cleaner module for HTML preprocessing (remove ads, base64 images, scripts)
- Media localizer now handles base64 data URIs alongside remote URLs
- Capture finalUrl from browser to track redirects for output path
- Agent quality gate documentation for post-capture validation
- Upgrade defuddle ^0.12.0 → ^0.14.0
- Add unit tests for content-cleaner, html-to-markdown, legacy-converter, media-localizer
This commit is contained in:
Jim Liu 宝玉
2026-03-24 22:39:17 -05:00
parent 40f9f05c22
commit e99ce744cd
12 changed files with 909 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/baoyu-url-to-markdown/scripts/html-to-markdown.ts
+28 -7
View File
@@ -13,10 +13,15 @@ import {
shouldCompareWithLegacy,
} from "./legacy-converter.js";
import { tryUrlRuleParsers } from "./parsers/index.js";
import { cleanContent } from "./content-cleaner.js";
export type { ConversionResult, PageMetadata };
export { createMarkdownDocument, formatMetadataYaml };
export interface ExtractContentOptions {
preserveBase64Images?: boolean;
}
export const absolutizeUrlsScript = String.raw`
(function() {
const baseUrl = document.baseURI || location.href;
@@ -85,7 +90,10 @@ export const absolutizeUrlsScript = String.raw`
absAttr(htmlClone, "video[poster]", "poster");
absSrcset(htmlClone, "img[srcset], source[srcset]");
return { html: "<!doctype html>\n" + htmlClone.outerHTML };
return {
html: "<!doctype html>\n" + htmlClone.outerHTML,
finalUrl: location.href,
};
})()
`;
@@ -102,7 +110,11 @@ function shouldPreferDefuddle(result: ConversionResult): boolean {
return /^##?\s+transcript\b/im.test(result.markdown);
}
export async function extractContent(html: string, url: string): Promise<ConversionResult> {
export async function extractContent(
html: string,
url: string,
options: ExtractContentOptions = {}
): Promise<ConversionResult> {
const capturedAt = new Date().toISOString();
const baseMetadata = extractMetadataFromHtml(html, url, capturedAt);
@@ -111,14 +123,23 @@ export async function extractContent(html: string, url: string): Promise<Convers
return specializedResult;
}
const defuddleResult = await tryDefuddleConversion(html, url, baseMetadata);
let cleanedHtml = html;
try {
cleanedHtml = cleanContent(html, url, {
removeBase64Images: !options.preserveBase64Images,
});
} catch {
cleanedHtml = html;
}
const defuddleResult = await tryDefuddleConversion(cleanedHtml, url, baseMetadata);
if (defuddleResult.ok) {
if (shouldPreferDefuddle(defuddleResult.result)) {
return defuddleResult.result;
return { ...defuddleResult.result, rawHtml: html };
}
if (shouldCompareWithLegacy(defuddleResult.result.markdown)) {
const legacyResult = convertWithLegacyExtractor(html, baseMetadata);
const legacyResult = convertWithLegacyExtractor(html, baseMetadata, cleanedHtml);
const legacyScore = scoreMarkdownQuality(legacyResult.markdown);
const defuddleScore = scoreMarkdownQuality(defuddleResult.result.markdown);
@@ -130,10 +151,10 @@ export async function extractContent(html: string, url: string): Promise<Convers
}
}
return defuddleResult.result;
return { ...defuddleResult.result, rawHtml: html };
}
const fallbackResult = convertWithLegacyExtractor(html, baseMetadata);
const fallbackResult = convertWithLegacyExtractor(html, baseMetadata, cleanedHtml);
return {
...fallbackResult,
fallbackReason: defuddleResult.reason,