mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-24 11:09:46 +08:00
feat(baoyu-fetch): add URL reader CLI with Chrome CDP and site adapters
This commit is contained in:
Jim Liu 宝玉
@@ -0,0 +1,51 @@
|
||||
export type ContentBlock =
|
||||
| {
|
||||
type: "paragraph";
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "heading";
|
||||
depth: number;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "list";
|
||||
ordered: boolean;
|
||||
items: string[];
|
||||
}
|
||||
| {
|
||||
type: "quote";
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "code";
|
||||
code: string;
|
||||
language?: string;
|
||||
}
|
||||
| {
|
||||
type: "image";
|
||||
url: string;
|
||||
alt?: string;
|
||||
}
|
||||
| {
|
||||
type: "html";
|
||||
html: string;
|
||||
}
|
||||
| {
|
||||
type: "markdown";
|
||||
markdown: string;
|
||||
};
|
||||
|
||||
export interface ExtractedDocument {
|
||||
url: string;
|
||||
requestedUrl?: string;
|
||||
canonicalUrl?: string;
|
||||
title?: string;
|
||||
author?: string;
|
||||
siteName?: string;
|
||||
publishedAt?: string;
|
||||
summary?: string;
|
||||
content: ContentBlock[];
|
||||
metadata?: Record<string, unknown>;
|
||||
adapter?: string;
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
|
||||
export interface CleanHtmlOptions {
|
||||
removeAds?: boolean;
|
||||
removeBase64Images?: boolean;
|
||||
onlyMainContent?: boolean;
|
||||
includeSelectors?: string[];
|
||||
excludeSelectors?: string[];
|
||||
}
|
||||
|
||||
const ALWAYS_REMOVE_SELECTORS = [
|
||||
"script",
|
||||
"style",
|
||||
"noscript",
|
||||
"link[rel='stylesheet']",
|
||||
"[hidden]",
|
||||
"[aria-hidden='true']",
|
||||
"[style*='display: none']",
|
||||
"[style*='display:none']",
|
||||
"[style*='visibility: hidden']",
|
||||
"[style*='visibility:hidden']",
|
||||
"svg[aria-hidden='true']",
|
||||
"svg.icon",
|
||||
"svg[class*='icon']",
|
||||
"template",
|
||||
"meta",
|
||||
"iframe",
|
||||
"canvas",
|
||||
"object",
|
||||
"embed",
|
||||
"form",
|
||||
"input",
|
||||
"select",
|
||||
"textarea",
|
||||
"button",
|
||||
];
|
||||
|
||||
const OVERLAY_SELECTORS = [
|
||||
"[class*='modal']",
|
||||
"[class*='popup']",
|
||||
"[class*='overlay']",
|
||||
"[class*='dialog']",
|
||||
"[role='dialog']",
|
||||
"[role='alertdialog']",
|
||||
"[class*='cookie']",
|
||||
"[class*='consent']",
|
||||
"[class*='gdpr']",
|
||||
"[class*='privacy-banner']",
|
||||
"[class*='notification-bar']",
|
||||
"[id*='cookie']",
|
||||
"[id*='consent']",
|
||||
"[id*='gdpr']",
|
||||
"[style*='position: fixed']",
|
||||
"[style*='position:fixed']",
|
||||
"[style*='position: sticky']",
|
||||
"[style*='position:sticky']",
|
||||
];
|
||||
|
||||
const NAVIGATION_SELECTORS = [
|
||||
"header",
|
||||
"footer",
|
||||
"nav",
|
||||
"aside",
|
||||
".header",
|
||||
".top",
|
||||
".navbar",
|
||||
"#header",
|
||||
".footer",
|
||||
".bottom",
|
||||
"#footer",
|
||||
".sidebar",
|
||||
".side",
|
||||
".aside",
|
||||
"#sidebar",
|
||||
".modal",
|
||||
".popup",
|
||||
"#modal",
|
||||
".overlay",
|
||||
".ad",
|
||||
".ads",
|
||||
".advert",
|
||||
"#ad",
|
||||
".lang-selector",
|
||||
".language",
|
||||
"#language-selector",
|
||||
".social",
|
||||
".social-media",
|
||||
".social-links",
|
||||
"#social",
|
||||
".menu",
|
||||
".navigation",
|
||||
"#nav",
|
||||
".breadcrumbs",
|
||||
"#breadcrumbs",
|
||||
".share",
|
||||
"#share",
|
||||
".widget",
|
||||
"#widget",
|
||||
".cookie",
|
||||
"#cookie",
|
||||
];
|
||||
|
||||
const FORCE_INCLUDE_SELECTORS = [
|
||||
"#main",
|
||||
"#content",
|
||||
"#main-content",
|
||||
"#article",
|
||||
"#post",
|
||||
"#page-content",
|
||||
"main",
|
||||
"article",
|
||||
"[role='main']",
|
||||
".main-content",
|
||||
".content",
|
||||
".post-content",
|
||||
".article-content",
|
||||
".entry-content",
|
||||
".page-content",
|
||||
".article-body",
|
||||
".post-body",
|
||||
".story-content",
|
||||
".blog-content",
|
||||
];
|
||||
|
||||
const AD_SELECTORS = [
|
||||
"ins.adsbygoogle",
|
||||
".google-ad",
|
||||
".adsense",
|
||||
"[data-ad]",
|
||||
"[data-ads]",
|
||||
"[data-ad-slot]",
|
||||
"[data-ad-client]",
|
||||
".ad-container",
|
||||
".ad-wrapper",
|
||||
".advertisement",
|
||||
".sponsored-content",
|
||||
"img[width='1'][height='1']",
|
||||
"img[src*='pixel']",
|
||||
"img[src*='tracking']",
|
||||
"img[src*='analytics']",
|
||||
];
|
||||
|
||||
function getLinkDensity(element: Element): number {
|
||||
const text = element.textContent || "";
|
||||
const textLength = text.trim().length;
|
||||
if (textLength === 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
let linkLength = 0;
|
||||
element.querySelectorAll("a").forEach((link) => {
|
||||
linkLength += (link.textContent || "").trim().length;
|
||||
});
|
||||
|
||||
return linkLength / textLength;
|
||||
}
|
||||
|
||||
function getContentScore(element: Element): number {
|
||||
let score = 0;
|
||||
const text = element.textContent || "";
|
||||
const textLength = text.trim().length;
|
||||
|
||||
score += Math.min(textLength / 100, 50);
|
||||
score += element.querySelectorAll("p").length * 3;
|
||||
score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2;
|
||||
score += element.querySelectorAll("img").length;
|
||||
|
||||
score -= element.querySelectorAll("a").length * 0.5;
|
||||
score -= element.querySelectorAll("li").length * 0.2;
|
||||
|
||||
const linkDensity = getLinkDensity(element);
|
||||
if (linkDensity > 0.5) {
|
||||
score -= 30;
|
||||
} else if (linkDensity > 0.3) {
|
||||
score -= 15;
|
||||
}
|
||||
|
||||
const className = typeof element.className === "string" ? element.className : "";
|
||||
const classAndId = `${className} ${element.id || ""}`;
|
||||
if (/article|content|post|body|main|entry/i.test(classAndId)) {
|
||||
score += 25;
|
||||
}
|
||||
if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) {
|
||||
score -= 25;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
function looksLikeNavigation(element: Element): boolean {
|
||||
const linkDensity = getLinkDensity(element);
|
||||
if (linkDensity > 0.5) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const listItems = element.querySelectorAll("li");
|
||||
const links = element.querySelectorAll("a");
|
||||
return listItems.length > 5 && links.length > listItems.length * 0.8;
|
||||
}
|
||||
|
||||
function removeElements(document: Document, selectors: string[]): void {
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
document.querySelectorAll(selector).forEach((element) => element.remove());
|
||||
} catch {
|
||||
// Ignore unsupported selectors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function removeWithProtection(
|
||||
document: Document,
|
||||
selectorsToRemove: string[],
|
||||
protectedSelectors: string[],
|
||||
): void {
|
||||
for (const selector of selectorsToRemove) {
|
||||
try {
|
||||
document.querySelectorAll(selector).forEach((element) => {
|
||||
const isProtected = protectedSelectors.some((protectedSelector) => {
|
||||
try {
|
||||
return element.matches(protectedSelector);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
if (isProtected) {
|
||||
return;
|
||||
}
|
||||
|
||||
const containsProtected = protectedSelectors.some((protectedSelector) => {
|
||||
try {
|
||||
return element.querySelector(protectedSelector) !== null;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
if (containsProtected) {
|
||||
return;
|
||||
}
|
||||
|
||||
element.remove();
|
||||
});
|
||||
} catch {
|
||||
// Ignore unsupported selectors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isValidContent(element: Element | null): element is Element {
|
||||
if (!element) {
|
||||
return false;
|
||||
}
|
||||
const text = element.textContent || "";
|
||||
if (text.trim().length < 100) {
|
||||
return false;
|
||||
}
|
||||
return !looksLikeNavigation(element);
|
||||
}
|
||||
|
||||
function findMainContent(document: Document): Element | null {
|
||||
const main = document.querySelector("main");
|
||||
if (isValidContent(main) && getLinkDensity(main) < 0.4) {
|
||||
return main;
|
||||
}
|
||||
|
||||
const roleMain = document.querySelector('[role="main"]');
|
||||
if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) {
|
||||
return roleMain;
|
||||
}
|
||||
|
||||
const articles = document.querySelectorAll("article");
|
||||
if (articles.length === 1 && isValidContent(articles[0] ?? null)) {
|
||||
return articles[0] ?? null;
|
||||
}
|
||||
|
||||
const contentSelectors = [
|
||||
"#content",
|
||||
"#main-content",
|
||||
"#main",
|
||||
".content",
|
||||
".main-content",
|
||||
".post-content",
|
||||
".article-content",
|
||||
".entry-content",
|
||||
".page-content",
|
||||
".article-body",
|
||||
".post-body",
|
||||
".story-content",
|
||||
".blog-content",
|
||||
];
|
||||
|
||||
for (const selector of contentSelectors) {
|
||||
try {
|
||||
const element = document.querySelector(selector);
|
||||
if (isValidContent(element) && getLinkDensity(element) < 0.4) {
|
||||
return element;
|
||||
}
|
||||
} catch {
|
||||
// Ignore invalid selectors.
|
||||
}
|
||||
}
|
||||
|
||||
const candidates: Array<{ element: Element; score: number }> = [];
|
||||
document.querySelectorAll("div, section, article").forEach((element) => {
|
||||
const text = element.textContent || "";
|
||||
if (text.trim().length < 200) {
|
||||
return;
|
||||
}
|
||||
|
||||
const score = getContentScore(element);
|
||||
if (score > 0) {
|
||||
candidates.push({ element, score });
|
||||
}
|
||||
});
|
||||
|
||||
candidates.sort((left, right) => right.score - left.score);
|
||||
if ((candidates[0]?.score ?? 0) > 20) {
|
||||
return candidates[0]?.element ?? null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function removeBase64ImagesFromDocument(document: Document): void {
|
||||
document.querySelectorAll("img[src^='data:']").forEach((element) => element.remove());
|
||||
|
||||
document.querySelectorAll("[style*='data:image']").forEach((element) => {
|
||||
const style = element.getAttribute("style");
|
||||
if (!style) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cleanedStyle = style.replace(
|
||||
/background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi,
|
||||
"",
|
||||
);
|
||||
|
||||
if (cleanedStyle.trim()) {
|
||||
element.setAttribute("style", cleanedStyle);
|
||||
} else {
|
||||
element.removeAttribute("style");
|
||||
}
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll("source[src^='data:'], source[srcset*='data:']")
|
||||
.forEach((element) => element.remove());
|
||||
}
|
||||
|
||||
function makeAbsoluteUrl(value: string, baseUrl: string): string | null {
|
||||
try {
|
||||
return new URL(value, baseUrl).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function convertRelativeUrls(document: Document, baseUrl: string): void {
|
||||
document.querySelectorAll("[src]").forEach((element) => {
|
||||
const src = element.getAttribute("src");
|
||||
if (!src || src.startsWith("http") || src.startsWith("//") || src.startsWith("data:")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const absolute = makeAbsoluteUrl(src, baseUrl);
|
||||
if (absolute) {
|
||||
element.setAttribute("src", absolute);
|
||||
}
|
||||
});
|
||||
|
||||
document.querySelectorAll("[href]").forEach((element) => {
|
||||
const href = element.getAttribute("href");
|
||||
if (
|
||||
!href ||
|
||||
href.startsWith("http") ||
|
||||
href.startsWith("//") ||
|
||||
href.startsWith("#") ||
|
||||
href.startsWith("mailto:") ||
|
||||
href.startsWith("tel:") ||
|
||||
href.startsWith("javascript:")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const absolute = makeAbsoluteUrl(href, baseUrl);
|
||||
if (absolute) {
|
||||
element.setAttribute("href", absolute);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function removeComments(document: Document): void {
|
||||
const walker = document.createTreeWalker(document, document.defaultView?.NodeFilter.SHOW_COMMENT ?? 128);
|
||||
const comments: Comment[] = [];
|
||||
while (walker.nextNode()) {
|
||||
comments.push(walker.currentNode as Comment);
|
||||
}
|
||||
comments.forEach((comment) => comment.parentNode?.removeChild(comment));
|
||||
}
|
||||
|
||||
export function cleanHtml(
|
||||
html: string,
|
||||
baseUrl: string,
|
||||
options: CleanHtmlOptions = {},
|
||||
): string {
|
||||
const {
|
||||
removeAds = true,
|
||||
removeBase64Images = true,
|
||||
onlyMainContent = true,
|
||||
includeSelectors,
|
||||
excludeSelectors,
|
||||
} = options;
|
||||
|
||||
const dom = new JSDOM(html, { url: baseUrl });
|
||||
const { document } = dom.window;
|
||||
|
||||
removeElements(document, ALWAYS_REMOVE_SELECTORS);
|
||||
removeElements(document, OVERLAY_SELECTORS);
|
||||
|
||||
if (removeAds) {
|
||||
removeElements(document, AD_SELECTORS);
|
||||
}
|
||||
|
||||
if (excludeSelectors?.length) {
|
||||
removeElements(document, excludeSelectors);
|
||||
}
|
||||
|
||||
if (onlyMainContent) {
|
||||
removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS);
|
||||
|
||||
const mainContent = findMainContent(document);
|
||||
if (mainContent && document.body) {
|
||||
const clone = mainContent.cloneNode(true);
|
||||
document.body.innerHTML = "";
|
||||
document.body.appendChild(clone);
|
||||
}
|
||||
}
|
||||
|
||||
if (includeSelectors?.length && document.body) {
|
||||
const matchedElements: Element[] = [];
|
||||
for (const selector of includeSelectors) {
|
||||
try {
|
||||
document.querySelectorAll(selector).forEach((element) => {
|
||||
matchedElements.push(element.cloneNode(true) as Element);
|
||||
});
|
||||
} catch {
|
||||
// Ignore invalid selectors.
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedElements.length > 0) {
|
||||
document.body.innerHTML = "";
|
||||
matchedElements.forEach((element) => document.body?.appendChild(element));
|
||||
}
|
||||
}
|
||||
|
||||
if (removeBase64Images) {
|
||||
removeBase64ImagesFromDocument(document);
|
||||
}
|
||||
|
||||
removeComments(document);
|
||||
convertRelativeUrls(document, baseUrl);
|
||||
|
||||
return document.documentElement.outerHTML || html;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { JSDOM } from "jsdom";
|
||||
import type { ExtractedDocument } from "./document";
|
||||
|
||||
function getMetaContent(document: Document, selectors: string[]): string | undefined {
|
||||
for (const selector of selectors) {
|
||||
const value = document.querySelector(selector)?.getAttribute("content")?.trim();
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function extractDocumentFromHtml(input: {
|
||||
url: string;
|
||||
html: string;
|
||||
adapter?: string;
|
||||
}): ExtractedDocument {
|
||||
const dom = new JSDOM(input.html, { url: input.url });
|
||||
const document = dom.window.document;
|
||||
|
||||
const canonicalUrl =
|
||||
document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
|
||||
getMetaContent(document, ['meta[property="og:url"]']);
|
||||
|
||||
const siteName = getMetaContent(document, [
|
||||
'meta[property="og:site_name"]',
|
||||
'meta[name="application-name"]',
|
||||
]);
|
||||
|
||||
const metadataAuthor = getMetaContent(document, [
|
||||
'meta[name="author"]',
|
||||
'meta[property="article:author"]',
|
||||
'meta[name="twitter:creator"]',
|
||||
]);
|
||||
|
||||
const publishedAt = getMetaContent(document, [
|
||||
'meta[property="article:published_time"]',
|
||||
'meta[name="pubdate"]',
|
||||
'meta[name="date"]',
|
||||
'meta[itemprop="datePublished"]',
|
||||
]);
|
||||
|
||||
const article = new Readability(document).parse();
|
||||
const title =
|
||||
article?.title?.trim() ||
|
||||
getMetaContent(document, ['meta[property="og:title"]']) ||
|
||||
document.title.trim() ||
|
||||
undefined;
|
||||
|
||||
const summary =
|
||||
article?.excerpt?.trim() ||
|
||||
getMetaContent(document, [
|
||||
'meta[name="description"]',
|
||||
'meta[property="og:description"]',
|
||||
'meta[name="twitter:description"]',
|
||||
]);
|
||||
|
||||
const contentHtml =
|
||||
article?.content?.trim() ||
|
||||
document.querySelector("main")?.innerHTML?.trim() ||
|
||||
document.body?.innerHTML?.trim() ||
|
||||
"";
|
||||
|
||||
const author = article?.byline?.trim() || metadataAuthor;
|
||||
|
||||
return {
|
||||
url: input.url,
|
||||
canonicalUrl,
|
||||
title,
|
||||
author,
|
||||
siteName,
|
||||
publishedAt,
|
||||
summary,
|
||||
adapter: input.adapter ?? "generic",
|
||||
metadata: {
|
||||
language: document.documentElement.lang || undefined,
|
||||
},
|
||||
content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,758 @@
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { Defuddle } from "defuddle/node";
|
||||
import { JSDOM, VirtualConsole } from "jsdom";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
import { collectMediaFromMarkdown } from "../media/markdown-media";
|
||||
import type { MediaAsset } from "../media/types";
|
||||
import { cleanHtml } from "./html-cleaner";
|
||||
|
||||
export interface HtmlConversionMetadata {
|
||||
url: string;
|
||||
canonicalUrl?: string;
|
||||
siteName?: string;
|
||||
title?: string;
|
||||
summary?: string;
|
||||
author?: string;
|
||||
publishedAt?: string;
|
||||
coverImage?: string;
|
||||
language?: string;
|
||||
capturedAt: string;
|
||||
}
|
||||
|
||||
export interface ConvertHtmlToMarkdownOptions {
|
||||
enableRemoteMarkdownFallback?: boolean;
|
||||
preserveBase64Images?: boolean;
|
||||
}
|
||||
|
||||
export interface HtmlToMarkdownResult {
|
||||
metadata: HtmlConversionMetadata;
|
||||
markdown: string;
|
||||
rawHtml: string;
|
||||
cleanedHtml: string;
|
||||
media: MediaAsset[];
|
||||
conversionMethod: string;
|
||||
fallbackReason?: string;
|
||||
}
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
const MIN_CONTENT_LENGTH = 120;
|
||||
const DEFUDDLE_API_ORIGIN = "https://defuddle.md";
|
||||
const LOCAL_FALLBACK_SCORE_DELTA = 120;
|
||||
const REMOTE_FALLBACK_SCORE_DELTA = 20;
|
||||
const LOW_QUALITY_MARKERS = [
|
||||
/Join The Conversation/i,
|
||||
/One Community\. Many Voices/i,
|
||||
/Read our community guidelines/i,
|
||||
/Create a free account to share your thoughts/i,
|
||||
/Become a Forbes Member/i,
|
||||
/Subscribe to trusted journalism/i,
|
||||
/\bComments\b/i,
|
||||
];
|
||||
|
||||
const ARTICLE_TYPES = new Set([
|
||||
"Article",
|
||||
"NewsArticle",
|
||||
"BlogPosting",
|
||||
"WebPage",
|
||||
"ReportageNewsArticle",
|
||||
]);
|
||||
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: "atx",
|
||||
bulletListMarker: "-",
|
||||
codeBlockStyle: "fenced",
|
||||
}) as TurndownService & {
|
||||
remove(selectors: string[]): void;
|
||||
addRule(
|
||||
key: string,
|
||||
rule: {
|
||||
filter: string | ((node: Node) => boolean);
|
||||
replacement: (content: string) => string;
|
||||
},
|
||||
): void;
|
||||
};
|
||||
|
||||
turndown.use(gfm);
|
||||
turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
|
||||
turndown.addRule("collapseFigure", {
|
||||
filter: "figure",
|
||||
replacement(content: string) {
|
||||
return `\n\n${content.trim()}\n\n`;
|
||||
},
|
||||
});
|
||||
turndown.addRule("dropInvisibleAnchors", {
|
||||
filter(node: Node) {
|
||||
return (
|
||||
node.nodeName === "A" &&
|
||||
!(node as Element).textContent?.trim() &&
|
||||
!(node as Element).querySelector("img, video, picture, source")
|
||||
);
|
||||
},
|
||||
replacement() {
|
||||
return "";
|
||||
},
|
||||
});
|
||||
|
||||
function pickString(...values: unknown[]): string | undefined {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeMarkdown(markdown: string): string {
|
||||
return markdown
|
||||
.replace(/\r\n/g, "\n")
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function stripWrappingQuotes(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (
|
||||
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
||||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
||||
) {
|
||||
return trimmed.slice(1, -1).trim();
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function stripMarkdownFrontmatter(markdown: string): string {
|
||||
return markdown.replace(/^\uFEFF?---\n[\s\S]*?\n---(?:\n|$)/, "").trim();
|
||||
}
|
||||
|
||||
function cleanMarkdownTitle(value: string): string | undefined {
|
||||
const cleaned = stripWrappingQuotes(
|
||||
value
|
||||
.replace(/\s+#+\s*$/, "")
|
||||
.replace(/!\[[^\]]*\]\([^)]+\)/g, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.replace(/[*_`~]/g, "")
|
||||
.trim(),
|
||||
);
|
||||
|
||||
return cleaned || undefined;
|
||||
}
|
||||
|
||||
export function extractTitleFromMarkdownDocument(markdown: string): string | undefined {
|
||||
const normalized = markdown.replace(/\r\n/g, "\n").trim();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const frontmatterMatch = normalized.match(/^\uFEFF?---\n([\s\S]*?)\n---(?:\n|$)/);
|
||||
if (frontmatterMatch) {
|
||||
for (const line of frontmatterMatch[1].split("\n")) {
|
||||
const match = line.match(/^title:\s*(.+?)\s*$/i);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const title = cleanMarkdownTitle(match[1]);
|
||||
if (title) {
|
||||
return title;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const body = stripMarkdownFrontmatter(normalized);
|
||||
const headingMatch = body.match(/^#{1,6}\s+(.+)$/m);
|
||||
if (!headingMatch) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return cleanMarkdownTitle(headingMatch[1]);
|
||||
}
|
||||
|
||||
function trimKnownBoilerplate(markdown: string): string {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
const lines = normalized.split("\n");
|
||||
|
||||
while (lines.length > 0) {
|
||||
const lastLine = lines[lines.length - 1]?.trim();
|
||||
if (!lastLine) {
|
||||
lines.pop();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^继续滑动看下一个$/.test(lastLine) || /^轻触阅读原文$/.test(lastLine)) {
|
||||
lines.pop();
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return normalizeMarkdown(lines.join("\n"));
|
||||
}
|
||||
|
||||
function buildDefuddleApiUrl(targetUrl: string): string {
|
||||
return `${DEFUDDLE_API_ORIGIN}/${encodeURIComponent(targetUrl)}`;
|
||||
}
|
||||
|
||||
async function fetchDefuddleApiMarkdown(
|
||||
targetUrl: string,
|
||||
): Promise<{ markdown: string; title?: string }> {
|
||||
const response = await fetch(buildDefuddleApiUrl(targetUrl), {
|
||||
headers: {
|
||||
accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
|
||||
},
|
||||
redirect: "follow",
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const rawMarkdown = (await response.text()).replace(/\r\n/g, "\n").trim();
|
||||
if (!rawMarkdown) {
|
||||
throw new Error("defuddle.md returned empty markdown");
|
||||
}
|
||||
|
||||
const title = extractTitleFromMarkdownDocument(rawMarkdown);
|
||||
const markdown = trimKnownBoilerplate(stripMarkdownFrontmatter(rawMarkdown));
|
||||
if (!markdown) {
|
||||
throw new Error("defuddle.md returned empty markdown");
|
||||
}
|
||||
|
||||
return {
|
||||
markdown,
|
||||
title,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeHtmlFragment(html: string): string {
|
||||
const dom = new JSDOM(`<div id="__root">${html}</div>`);
|
||||
const root = dom.window.document.querySelector("#__root");
|
||||
if (!root) {
|
||||
return html;
|
||||
}
|
||||
|
||||
for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
|
||||
root.querySelectorAll(selector).forEach((element) => element.remove());
|
||||
}
|
||||
|
||||
return root.innerHTML;
|
||||
}
|
||||
|
||||
function extractTextFromHtml(html: string): string {
|
||||
const dom = new JSDOM(`<!doctype html><html><body>${html}</body></html>`);
|
||||
const { document } = dom.window;
|
||||
for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
|
||||
document.querySelectorAll(selector).forEach((element) => element.remove());
|
||||
}
|
||||
return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
|
||||
}
|
||||
|
||||
function getMetaContent(document: Document, names: string[]): string | undefined {
|
||||
for (const name of names) {
|
||||
const element =
|
||||
document.querySelector(`meta[name="${name}"]`) ??
|
||||
document.querySelector(`meta[property="${name}"]`);
|
||||
const content = element?.getAttribute("content")?.trim();
|
||||
if (content) {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeLanguageTag(value: string | null | undefined): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim();
|
||||
if (!primary) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return primary.replace(/_/g, "-");
|
||||
}
|
||||
|
||||
function flattenJsonLdItems(data: unknown): JsonObject[] {
|
||||
if (!data || typeof data !== "object") {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
return data.flatMap(flattenJsonLdItems);
|
||||
}
|
||||
|
||||
const item = data as JsonObject;
|
||||
if (Array.isArray(item["@graph"])) {
|
||||
return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
|
||||
}
|
||||
|
||||
return [item];
|
||||
}
|
||||
|
||||
function parseJsonLdScripts(document: Document): JsonObject[] {
|
||||
const results: JsonObject[] = [];
|
||||
document.querySelectorAll("script[type='application/ld+json']").forEach((script) => {
|
||||
try {
|
||||
const data = JSON.parse(script.textContent ?? "");
|
||||
results.push(...flattenJsonLdItems(data));
|
||||
} catch {
|
||||
// Ignore malformed json-ld blocks.
|
||||
}
|
||||
});
|
||||
return results;
|
||||
}
|
||||
|
||||
function extractAuthorFromJsonLd(authorData: unknown): string | undefined {
|
||||
if (typeof authorData === "string") {
|
||||
return authorData.trim() || undefined;
|
||||
}
|
||||
|
||||
if (!authorData || typeof authorData !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (Array.isArray(authorData)) {
|
||||
return authorData
|
||||
.map((author) => extractAuthorFromJsonLd(author))
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join(", ") || undefined;
|
||||
}
|
||||
|
||||
const author = authorData as JsonObject;
|
||||
return pickString(author.name);
|
||||
}
|
||||
|
||||
function extractPrimaryJsonLdMeta(document: Document): Partial<HtmlConversionMetadata> {
|
||||
for (const item of parseJsonLdScripts(document)) {
|
||||
const type = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
|
||||
if (typeof type !== "string" || !ARTICLE_TYPES.has(type)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
title: pickString(item.headline, item.name),
|
||||
summary: pickString(item.description),
|
||||
author: extractAuthorFromJsonLd(item.author),
|
||||
publishedAt: pickString(item.datePublished, item.dateCreated),
|
||||
coverImage: pickString(
|
||||
item.image,
|
||||
(item.image as JsonObject | undefined)?.url,
|
||||
Array.isArray(item.image) ? item.image[0] : undefined,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
function extractPageMetadata(
|
||||
html: string,
|
||||
url: string,
|
||||
capturedAt: string,
|
||||
): HtmlConversionMetadata {
|
||||
const dom = new JSDOM(html, { url });
|
||||
const { document } = dom.window;
|
||||
const jsonLd = extractPrimaryJsonLdMeta(document);
|
||||
|
||||
return {
|
||||
url,
|
||||
canonicalUrl:
|
||||
document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
|
||||
getMetaContent(document, ["og:url"]),
|
||||
siteName: pickString(
|
||||
getMetaContent(document, ["og:site_name"]),
|
||||
document.querySelector('meta[name="application-name"]')?.getAttribute("content"),
|
||||
),
|
||||
title: pickString(
|
||||
getMetaContent(document, ["og:title", "twitter:title"]),
|
||||
jsonLd.title,
|
||||
document.querySelector("h1")?.textContent,
|
||||
document.title,
|
||||
),
|
||||
summary: pickString(
|
||||
getMetaContent(document, ["description", "og:description", "twitter:description"]),
|
||||
jsonLd.summary,
|
||||
),
|
||||
author: pickString(
|
||||
getMetaContent(document, ["author", "article:author", "twitter:creator"]),
|
||||
jsonLd.author,
|
||||
),
|
||||
publishedAt: pickString(
|
||||
document.querySelector("time[datetime]")?.getAttribute("datetime"),
|
||||
getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
|
||||
jsonLd.publishedAt,
|
||||
),
|
||||
coverImage: pickString(
|
||||
getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
|
||||
jsonLd.coverImage,
|
||||
),
|
||||
language: pickString(
|
||||
normalizeLanguageTag(document.documentElement.getAttribute("lang")),
|
||||
normalizeLanguageTag(
|
||||
pickString(
|
||||
getMetaContent(document, ["language", "content-language", "og:locale"]),
|
||||
document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content"),
|
||||
),
|
||||
),
|
||||
),
|
||||
capturedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function isMarkdownUsable(markdown: string, html: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const htmlTextLength = extractTextFromHtml(html).length;
|
||||
if (htmlTextLength < MIN_CONTENT_LENGTH) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.length >= 80) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
|
||||
}
|
||||
|
||||
function countMarkerHits(markdown: string, markers: RegExp[]): number {
|
||||
let hits = 0;
|
||||
for (const marker of markers) {
|
||||
if (marker.test(markdown)) {
|
||||
hits += 1;
|
||||
}
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
function countUsefulParagraphs(markdown: string): number {
|
||||
const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/);
|
||||
let count = 0;
|
||||
|
||||
for (const paragraph of paragraphs) {
|
||||
const trimmed = paragraph.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
if (/^#{1,6}\s+/.test(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) {
|
||||
continue;
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
function scoreMarkdownQuality(markdown: string): number {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length;
|
||||
const usefulParagraphs = countUsefulParagraphs(normalized);
|
||||
const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length;
|
||||
const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS);
|
||||
return Math.min(wordCount, 4000) + usefulParagraphs * 40 + headingCount * 10 - markerHits * 180;
|
||||
}
|
||||
|
||||
function shouldCompareWithFallback(markdown: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
return countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 || countUsefulParagraphs(normalized) < 6;
|
||||
}
|
||||
|
||||
function hasMeaningfulMarkdownStructure(markdown: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
countUsefulParagraphs(normalized) > 0 ||
|
||||
/^#{1,6}\s+/m.test(normalized) ||
|
||||
/^[-*]\s+/m.test(normalized) ||
|
||||
/^\d+\.\s+/m.test(normalized) ||
|
||||
/!\[[^\]]*\]\([^)]+\)/.test(normalized)
|
||||
);
|
||||
}
|
||||
|
||||
function shouldTryRemoteMarkdownFallback(
|
||||
markdown: string,
|
||||
html: string,
|
||||
options: ConvertHtmlToMarkdownOptions,
|
||||
): boolean {
|
||||
if (!options.enableRemoteMarkdownFallback) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !isMarkdownUsable(markdown, html) || shouldCompareWithFallback(markdown);
|
||||
}
|
||||
|
||||
function shouldPreferRemoteMarkdown(
|
||||
current: HtmlToMarkdownResult,
|
||||
remote: HtmlToMarkdownResult,
|
||||
html: string,
|
||||
): boolean {
|
||||
if (!isMarkdownUsable(current.markdown, html)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!hasMeaningfulMarkdownStructure(current.markdown) && hasMeaningfulMarkdownStructure(remote.markdown)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return scoreMarkdownQuality(remote.markdown) > scoreMarkdownQuality(current.markdown) + REMOTE_FALLBACK_SCORE_DELTA;
|
||||
}
|
||||
|
||||
function buildRemoteFallbackReason(current: HtmlToMarkdownResult, html: string): string {
|
||||
if (!isMarkdownUsable(current.markdown, html)) {
|
||||
return current.fallbackReason
|
||||
? `Used defuddle.md markdown fallback after local extraction failed: ${current.fallbackReason}`
|
||||
: "Used defuddle.md markdown fallback after local extraction returned empty or incomplete markdown";
|
||||
}
|
||||
|
||||
return "defuddle.md produced higher-quality markdown than local extraction";
|
||||
}
|
||||
|
||||
async function tryDefuddleConversion(
|
||||
html: string,
|
||||
url: string,
|
||||
baseMetadata: HtmlConversionMetadata,
|
||||
): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
|
||||
try {
|
||||
const virtualConsole = new VirtualConsole();
|
||||
virtualConsole.on("jsdomError", (error: Error & { type?: string }) => {
|
||||
if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) {
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
const dom = new JSDOM(html, { url, virtualConsole });
|
||||
const result = await Defuddle(dom, url, { markdown: true });
|
||||
const markdown = trimKnownBoilerplate(result.content || "");
|
||||
|
||||
if (!isMarkdownUsable(markdown, html)) {
|
||||
return { ok: false, reason: "Defuddle returned empty or incomplete markdown" };
|
||||
}
|
||||
|
||||
const metadata: HtmlConversionMetadata = {
|
||||
...baseMetadata,
|
||||
title: pickString(result.title, baseMetadata.title),
|
||||
summary: pickString(result.description, baseMetadata.summary),
|
||||
author: pickString(result.author, baseMetadata.author),
|
||||
publishedAt: pickString(result.published, baseMetadata.publishedAt),
|
||||
coverImage: pickString(result.image, baseMetadata.coverImage),
|
||||
language: pickString(result.language, baseMetadata.language),
|
||||
};
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
metadata,
|
||||
markdown,
|
||||
rawHtml: html,
|
||||
cleanedHtml: html,
|
||||
media: collectMediaFromMarkdown(markdown).concat(
|
||||
metadata.coverImage
|
||||
? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
|
||||
: [],
|
||||
),
|
||||
conversionMethod: "defuddle",
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function tryDefuddleApiConversion(
|
||||
html: string,
|
||||
url: string,
|
||||
baseMetadata: HtmlConversionMetadata,
|
||||
): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
|
||||
try {
|
||||
const result = await fetchDefuddleApiMarkdown(url);
|
||||
const markdown = result.markdown;
|
||||
|
||||
if (!isMarkdownUsable(markdown, html) && scoreMarkdownQuality(markdown) < 80) {
|
||||
return { ok: false, reason: "defuddle.md returned empty or incomplete markdown" };
|
||||
}
|
||||
|
||||
const metadata: HtmlConversionMetadata = {
|
||||
...baseMetadata,
|
||||
title: pickString(result.title, baseMetadata.title),
|
||||
};
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
metadata,
|
||||
markdown,
|
||||
rawHtml: html,
|
||||
cleanedHtml: html,
|
||||
media: collectMediaFromMarkdown(markdown).concat(
|
||||
metadata.coverImage
|
||||
? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
|
||||
: [],
|
||||
),
|
||||
conversionMethod: "defuddle-api",
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function convertHtmlFragmentToMarkdown(html: string): string {
|
||||
if (!html.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
return turndown.turndown(sanitizeHtmlFragment(html));
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackPlainText(html: string): string {
|
||||
return trimKnownBoilerplate(extractTextFromHtml(html));
|
||||
}
|
||||
|
||||
function convertWithReadability(
|
||||
rawHtml: string,
|
||||
cleanedHtml: string,
|
||||
url: string,
|
||||
baseMetadata: HtmlConversionMetadata,
|
||||
): HtmlToMarkdownResult {
|
||||
const dom = new JSDOM(cleanedHtml, { url });
|
||||
const document = dom.window.document;
|
||||
const article = new Readability(document).parse();
|
||||
|
||||
const contentHtml =
|
||||
article?.content?.trim() ??
|
||||
document.querySelector("main")?.innerHTML?.trim() ??
|
||||
document.body?.innerHTML?.trim() ??
|
||||
"";
|
||||
|
||||
let markdown = contentHtml ? convertHtmlFragmentToMarkdown(contentHtml) : "";
|
||||
if (!markdown) {
|
||||
markdown = fallbackPlainText(cleanedHtml);
|
||||
}
|
||||
|
||||
const metadata: HtmlConversionMetadata = {
|
||||
...baseMetadata,
|
||||
title: pickString(article?.title, baseMetadata.title),
|
||||
summary: pickString(article?.excerpt, baseMetadata.summary),
|
||||
author: pickString(article?.byline, baseMetadata.author),
|
||||
};
|
||||
|
||||
const media = collectMediaFromMarkdown(markdown);
|
||||
if (metadata.coverImage) {
|
||||
media.unshift({
|
||||
url: metadata.coverImage,
|
||||
kind: "image",
|
||||
role: "cover",
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
metadata,
|
||||
markdown: trimKnownBoilerplate(markdown),
|
||||
rawHtml,
|
||||
cleanedHtml,
|
||||
media,
|
||||
conversionMethod: article?.content ? "legacy:readability" : "legacy:body",
|
||||
};
|
||||
}
|
||||
|
||||
export async function convertHtmlToMarkdown(
|
||||
html: string,
|
||||
url: string,
|
||||
options: ConvertHtmlToMarkdownOptions = {},
|
||||
): Promise<HtmlToMarkdownResult> {
|
||||
const capturedAt = new Date().toISOString();
|
||||
const baseMetadata = extractPageMetadata(html, url, capturedAt);
|
||||
|
||||
let cleanedHtml = html;
|
||||
try {
|
||||
cleanedHtml = cleanHtml(html, url, {
|
||||
removeBase64Images: !options.preserveBase64Images,
|
||||
});
|
||||
} catch {
|
||||
cleanedHtml = html;
|
||||
}
|
||||
|
||||
let selectedResult: HtmlToMarkdownResult;
|
||||
const defuddleResult = await tryDefuddleConversion(cleanedHtml, url, baseMetadata);
|
||||
if (defuddleResult.ok) {
|
||||
if (shouldCompareWithFallback(defuddleResult.result.markdown)) {
|
||||
const fallbackResult = convertWithReadability(html, cleanedHtml, url, baseMetadata);
|
||||
if (
|
||||
scoreMarkdownQuality(fallbackResult.markdown) >
|
||||
scoreMarkdownQuality(defuddleResult.result.markdown) + LOCAL_FALLBACK_SCORE_DELTA
|
||||
) {
|
||||
selectedResult = {
|
||||
...fallbackResult,
|
||||
fallbackReason: "Readability/Turndown produced higher-quality markdown than Defuddle",
|
||||
};
|
||||
} else {
|
||||
selectedResult = {
|
||||
...defuddleResult.result,
|
||||
rawHtml: html,
|
||||
cleanedHtml,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
selectedResult = {
|
||||
...defuddleResult.result,
|
||||
rawHtml: html,
|
||||
cleanedHtml,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
selectedResult = {
|
||||
...convertWithReadability(html, cleanedHtml, url, baseMetadata),
|
||||
fallbackReason: defuddleResult.reason,
|
||||
};
|
||||
}
|
||||
|
||||
if (!shouldTryRemoteMarkdownFallback(selectedResult.markdown, cleanedHtml, options)) {
|
||||
return selectedResult;
|
||||
}
|
||||
|
||||
const remoteDefuddleResult = await tryDefuddleApiConversion(cleanedHtml, url, baseMetadata);
|
||||
if (!remoteDefuddleResult.ok || !shouldPreferRemoteMarkdown(selectedResult, remoteDefuddleResult.result, cleanedHtml)) {
|
||||
return selectedResult;
|
||||
}
|
||||
|
||||
return {
|
||||
...remoteDefuddleResult.result,
|
||||
rawHtml: html,
|
||||
cleanedHtml,
|
||||
fallbackReason: buildRemoteFallbackReason(selectedResult, cleanedHtml),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
import { normalizeMarkdownMediaLinks } from "../media/markdown-media";
|
||||
import type { ContentBlock, ExtractedDocument } from "./document";
|
||||
|
||||
const turndownService = new TurndownService({
|
||||
codeBlockStyle: "fenced",
|
||||
headingStyle: "atx",
|
||||
bulletListMarker: "-",
|
||||
});
|
||||
|
||||
turndownService.use(gfm);
|
||||
|
||||
function renderBlock(block: ContentBlock): string {
|
||||
switch (block.type) {
|
||||
case "paragraph":
|
||||
return block.text.trim();
|
||||
case "heading":
|
||||
return `${"#".repeat(Math.min(Math.max(block.depth, 1), 6))} ${block.text.trim()}`;
|
||||
case "list":
|
||||
return block.items
|
||||
.map((item, index) => (block.ordered ? `${index + 1}. ${item.trim()}` : `- ${item.trim()}`))
|
||||
.join("\n");
|
||||
case "quote":
|
||||
return block.text
|
||||
.split("\n")
|
||||
.map((line) => `> ${line}`)
|
||||
.join("\n");
|
||||
case "code":
|
||||
return `\`\`\`${block.language ?? ""}\n${block.code.trimEnd()}\n\`\`\``;
|
||||
case "image":
|
||||
return ``;
|
||||
case "html":
|
||||
return turndownService.turndown(block.html).trim();
|
||||
case "markdown":
|
||||
return block.markdown.trim();
|
||||
}
|
||||
}
|
||||
|
||||
function isDefinedValue(value: unknown): boolean {
|
||||
return value !== undefined && value !== null && value !== "";
|
||||
}
|
||||
|
||||
function renderFrontmatterValue(value: unknown): string {
|
||||
if (typeof value === "string") {
|
||||
if (value.includes("\n")) {
|
||||
return `|-\n${value
|
||||
.replace(/\r\n/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n")}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
if (typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function renderFrontmatter(document: ExtractedDocument): string {
|
||||
const fields = new Map<string, unknown>();
|
||||
const preferredOrder = [
|
||||
"title",
|
||||
"url",
|
||||
"requestedUrl",
|
||||
"author",
|
||||
"authorName",
|
||||
"authorUsername",
|
||||
"authorUrl",
|
||||
"coverImage",
|
||||
"siteName",
|
||||
"publishedAt",
|
||||
"summary",
|
||||
"adapter",
|
||||
];
|
||||
|
||||
fields.set("title", document.title);
|
||||
fields.set("url", document.canonicalUrl ?? document.url);
|
||||
fields.set("requestedUrl", document.requestedUrl ?? document.url);
|
||||
fields.set("author", document.author);
|
||||
fields.set("siteName", document.siteName);
|
||||
fields.set("publishedAt", document.publishedAt);
|
||||
fields.set("summary", document.summary);
|
||||
fields.set("adapter", document.adapter);
|
||||
|
||||
for (const [key, value] of Object.entries(document.metadata ?? {})) {
|
||||
if (!fields.has(key)) {
|
||||
fields.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
const orderedKeys = [
|
||||
...preferredOrder.filter((key) => fields.has(key)),
|
||||
...Array.from(fields.keys()).filter((key) => !preferredOrder.includes(key)).sort(),
|
||||
];
|
||||
|
||||
const lines = orderedKeys
|
||||
.map((key) => [key, fields.get(key)] as const)
|
||||
.filter(([, value]) => isDefinedValue(value))
|
||||
.map(([key, value]) => `${key}: ${renderFrontmatterValue(value)}`);
|
||||
|
||||
if (lines.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return `---\n${lines.join("\n")}\n---`;
|
||||
}
|
||||
|
||||
function cleanMarkdown(markdown: string): string {
|
||||
return normalizeMarkdownMediaLinks(markdown.replace(/\n{3,}/g, "\n\n").trim());
|
||||
}
|
||||
|
||||
function normalizeComparableTitle(value: string): string {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/^>\s*/, "")
|
||||
.replace(/^#+\s+/, "")
|
||||
.replace(/(?:\.{3}|…)\s*$/, "");
|
||||
}
|
||||
|
||||
function bodyStartsWithTitle(body: string, title: string): boolean {
|
||||
const firstMeaningfulLine = body
|
||||
.replace(/\r\n/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line));
|
||||
|
||||
if (!firstMeaningfulLine) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const comparableTitle = normalizeComparableTitle(title);
|
||||
const comparableFirstLine = normalizeComparableTitle(firstMeaningfulLine);
|
||||
if (!comparableTitle || !comparableFirstLine) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
comparableFirstLine === comparableTitle ||
|
||||
comparableFirstLine.startsWith(comparableTitle) ||
|
||||
comparableTitle.startsWith(comparableFirstLine)
|
||||
);
|
||||
}
|
||||
|
||||
export function renderMarkdown(document: ExtractedDocument): string {
|
||||
const sections: string[] = [];
|
||||
const frontmatter = renderFrontmatter(document);
|
||||
|
||||
if (frontmatter) {
|
||||
sections.push(frontmatter);
|
||||
}
|
||||
|
||||
const body = document.content
|
||||
.map((block) => renderBlock(block))
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
if (document.title && !bodyStartsWithTitle(body, document.title)) {
|
||||
sections.push(`# ${document.title}`);
|
||||
}
|
||||
|
||||
if (body) {
|
||||
sections.push(body);
|
||||
}
|
||||
|
||||
return cleanMarkdown(sections.join("\n\n"));
|
||||
}
|
||||
Reference in New Issue
Block a user