feat(baoyu-url-to-markdown): add URL-specific parser layer for X/Twitter and archive.ph

- New parsers/ module with pluggable rule system for site-specific HTML extraction
- X status parser: extract tweet text, media, quotes, author from data-testid elements
- X article parser: extract long-form article content with inline media
- archive.ph parser: restore original URL and prefer #CONTENT container
- Improved slug generation with stop words and content-aware slugs
- Output path uses subdirectory structure (domain/slug/slug.md)
- Fix: preserve anchor elements containing media in legacy converter
- Fix: smarter title deduplication in markdown document builder
This commit is contained in:
Jim Liu 宝玉
2026-03-22 15:18:46 -05:00
parent 6a4b312146
commit e5d6c8ec68
13 changed files with 953 additions and 26 deletions
@@ -12,6 +12,7 @@ import {
scoreMarkdownQuality,
shouldCompareWithLegacy,
} from "./legacy-converter.js";
import { tryUrlRuleParsers } from "./parsers/index.js";
export type { ConversionResult, PageMetadata };
export { createMarkdownDocument, formatMetadataYaml };
@@ -105,6 +106,11 @@ export async function extractContent(html: string, url: string): Promise<Convers
const capturedAt = new Date().toISOString();
const baseMetadata = extractMetadataFromHtml(html, url, capturedAt);
const specializedResult = tryUrlRuleParsers(html, url, baseMetadata);
if (specializedResult) {
return specializedResult;
}
const defuddleResult = await tryDefuddleConversion(html, url, baseMetadata);
if (defuddleResult.ok) {
if (shouldPreferDefuddle(defuddleResult.result)) {