From 832f06e86ea3501f011da8229fbd92601d77876b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20Liu=20=E5=AE=9D=E7=8E=89?= Date: Thu, 5 Mar 2026 00:11:59 -0600 Subject: [PATCH] refactor(baoyu-url-to-markdown): replace custom extraction with defuddle library --- .../scripts/html-to-markdown.ts | 829 +----------------- skills/baoyu-url-to-markdown/scripts/main.ts | 18 +- 2 files changed, 37 insertions(+), 810 deletions(-) diff --git a/skills/baoyu-url-to-markdown/scripts/html-to-markdown.ts b/skills/baoyu-url-to-markdown/scripts/html-to-markdown.ts index bf1f644..a684b23 100644 --- a/skills/baoyu-url-to-markdown/scripts/html-to-markdown.ts +++ b/skills/baoyu-url-to-markdown/scripts/html-to-markdown.ts @@ -1,7 +1,5 @@ -import { parseHTML } from "linkedom"; -import { Readability } from "@mozilla/readability"; -import TurndownService from "turndown"; -import { gfm } from "turndown-plugin-gfm"; +import { JSDOM } from "jsdom"; +import { Defuddle } from "defuddle/node"; export interface PageMetadata { url: string; @@ -17,812 +15,51 @@ export interface ConversionResult { markdown: string; } -interface ExtractionCandidate { - title: string | null; - byline: string | null; - excerpt: string | null; - published: string | null; - html: string | null; - textContent: string; - method: string; -} - -type AnyRecord = Record; - -const MIN_CONTENT_LENGTH = 120; -const GOOD_CONTENT_LENGTH = 900; - -const CONTENT_SELECTORS = [ - "article", - "main article", - "[role='main'] article", - "[itemprop='articleBody']", - ".article-content", - ".article-body", - ".post-content", - ".entry-content", - ".story-body", - "main", - "[role='main']", - "#content", - ".content", -]; - -const REMOVE_SELECTORS = [ - "script", - "style", - "noscript", - "template", - "iframe", - "svg", - "path", - "nav", - "aside", - "footer", - "header", - "form", - ".advertisement", - ".ads", - ".social-share", - ".related-articles", - ".comments", - ".newsletter", - ".cookie-banner", - ".cookie-consent", - "[role='navigation']", - "[aria-label*='cookie' i]", -]; - -const PUBLISHED_TIME_SELECTORS = [ - "meta[property='article:published_time']", - "meta[name='pubdate']", - "meta[name='publishdate']", - "meta[name='date']", - "time[datetime]", -]; - -const ARTICLE_TYPES = new Set([ - "Article", - "NewsArticle", - "BlogPosting", - "WebPage", - "ReportageNewsArticle", -]); - -const NEXT_DATA_CONTENT_PATHS = [ - "props.pageProps.content.body", - "props.pageProps.article.body", - "props.pageProps.article.content", - "props.pageProps.post.body", - "props.pageProps.post.content", - "props.pageProps.data.body", - "props.pageProps.story.body.content", -]; - -const cleanupAndExtractScriptBody = String.raw` -(function () { +export const absolutizeUrlsScript = String.raw` +(function() { const baseUrl = document.baseURI || location.href; - function toAbsolute(url) { if (!url) return url; - try { - return new URL(url, baseUrl).href; - } catch { - return url; - } + try { return new URL(url, baseUrl).href; } catch { return url; } } - - function absolutizeAttr(selector, attr) { - document.querySelectorAll(selector).forEach((el) => { - const value = el.getAttribute(attr); - if (!value) return; - const abs = toAbsolute(value); - if (abs) el.setAttribute(attr, abs); + function absAttr(sel, attr) { + document.querySelectorAll(sel).forEach(el => { + const v = el.getAttribute(attr); + if (v) { const a = toAbsolute(v); if (a) el.setAttribute(attr, a); } }); } - - function absolutizeSrcset(selector) { - document.querySelectorAll(selector).forEach((el) => { - const srcset = el.getAttribute("srcset"); - if (!srcset) return; - - const normalized = srcset - .split(",") - .map((part) => { - const trimmed = part.trim(); - if (!trimmed) return ""; - const pieces = trimmed.split(/\s+/); - const url = pieces[0]; - const descriptor = pieces.slice(1).join(" "); - const absoluteUrl = toAbsolute(url); - return descriptor ? absoluteUrl + " " + descriptor : absoluteUrl; - }) - .filter(Boolean) - .join(", "); - - if (normalized) { - el.setAttribute("srcset", normalized); - } + function absSrcset(sel) { + document.querySelectorAll(sel).forEach(el => { + const s = el.getAttribute("srcset"); + if (!s) return; + el.setAttribute("srcset", s.split(",").map(p => { + const t = p.trim(); if (!t) return ""; + const [url, ...d] = t.split(/\s+/); + return d.length ? toAbsolute(url) + " " + d.join(" ") : toAbsolute(url); + }).filter(Boolean).join(", ")); }); } - - absolutizeAttr("a[href]", "href"); - absolutizeAttr("img[src], video[src], audio[src], source[src]", "src"); - absolutizeSrcset("img[srcset], source[srcset]"); - - const removeSelectors = [ - "noscript", - "template", - ".cookie-banner", - ".cookie-consent", - ".consent-banner", - "[aria-label*='cookie' i]", - ".advertisement", - ".ads" - ]; - - for (const sel of removeSelectors) { - try { - document.querySelectorAll(sel).forEach((el) => el.remove()); - } catch {} - } - - function getMeta(names) { - for (const name of names) { - const el = document.querySelector('meta[name="' + name + '"]') || document.querySelector('meta[property="' + name + '"]'); - const content = el && el.getAttribute("content"); - if (content && content.trim()) return content.trim(); - } - return undefined; - } - - function flattenJsonLdItems(value) { - if (!value || typeof value !== "object") return []; - if (Array.isArray(value)) { - return value.flatMap(flattenJsonLdItems); - } - - const obj = value; - if (Array.isArray(obj["@graph"])) { - return obj["@graph"].flatMap(flattenJsonLdItems); - } - - return [obj]; - } - - function extractJsonLdMeta() { - const scripts = document.querySelectorAll('script[type="application/ld+json"]'); - for (const script of scripts) { - try { - const parsed = JSON.parse(script.textContent || ""); - const items = flattenJsonLdItems(parsed); - for (const item of items) { - const rawType = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"]; - if (typeof rawType !== "string") continue; - if (!["Article", "NewsArticle", "BlogPosting", "WebPage"].includes(rawType)) continue; - - const author = (() => { - if (typeof item.author === "string") return item.author; - if (Array.isArray(item.author) && item.author.length > 0) { - const first = item.author[0]; - return first && typeof first === "object" ? first.name : undefined; - } - if (item.author && typeof item.author === "object") { - return item.author.name; - } - return undefined; - })(); - - return { - title: item.headline || item.name, - description: item.description, - author: typeof author === "string" ? author : undefined, - published: item.datePublished || item.dateCreated, - }; - } - } catch {} - } - return {}; - } - - const jsonLd = extractJsonLdMeta(); - - const timeEl = document.querySelector("time[datetime]"); - const title = - getMeta(["og:title", "twitter:title"]) || - (typeof jsonLd.title === "string" ? jsonLd.title : undefined) || - document.querySelector("h1")?.textContent?.trim() || - document.title?.trim() || - ""; - - const description = - getMeta(["description", "og:description", "twitter:description"]) || - (typeof jsonLd.description === "string" ? jsonLd.description : undefined); - - const author = - getMeta(["author", "article:author", "twitter:creator"]) || - (typeof jsonLd.author === "string" ? jsonLd.author : undefined); - - const published = - timeEl?.getAttribute("datetime") || - getMeta(["article:published_time", "datePublished", "publishdate", "date"]) || - (typeof jsonLd.published === "string" ? jsonLd.published : undefined); - - return { - title, - description, - author, - published, - html: document.documentElement.outerHTML, - }; + absAttr("a[href]", "href"); + absAttr("img[src], video[src], audio[src], source[src]", "src"); + absSrcset("img[srcset], source[srcset]"); + return { html: document.documentElement.outerHTML }; })() `; -export const cleanupAndExtractScript = cleanupAndExtractScriptBody; +export async function extractContent(html: string, url: string): Promise { + const dom = new JSDOM(html, { url }); + const result = await Defuddle(dom, url, { markdown: true }); -function pickString(...values: unknown[]): string | null { - for (const value of values) { - if (typeof value === "string") { - const trimmed = value.trim(); - if (trimmed) return trimmed; - } - } - return null; -} - -function generateExcerpt(excerpt: string | null, textContent: string | null): string | null { - if (excerpt) return excerpt; - if (!textContent) return null; - const trimmed = textContent.trim(); - if (!trimmed) return null; - return trimmed.length > 200 ? `${trimmed.slice(0, 200)}...` : trimmed; -} - -function extractPublishedTime(document: any): string | null { - for (const selector of PUBLISHED_TIME_SELECTORS) { - const el = document.querySelector(selector); - if (!el) continue; - const value = el.getAttribute("content") ?? el.getAttribute("datetime"); - if (value && value.trim()) return value.trim(); - } - return null; -} - -function extractTitle(document: any): string | null { - const ogTitle = document.querySelector("meta[property='og:title']")?.getAttribute("content"); - if (ogTitle && ogTitle.trim()) return ogTitle.trim(); - - const twitterTitle = document - .querySelector("meta[name='twitter:title']") - ?.getAttribute("content"); - if (twitterTitle && twitterTitle.trim()) return twitterTitle.trim(); - - const title = document.querySelector("title")?.textContent?.trim(); - if (title) { - const cleaned = title.split(/\s*[-|–—]\s*/)[0]?.trim(); - if (cleaned) return cleaned; - } - - const h1 = document.querySelector("h1")?.textContent?.trim(); - return h1 || null; -} - -function extractTextFromHtml(html: string): string { - const { document } = parseHTML(`${html}`); - for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) { - for (const el of document.querySelectorAll(selector)) { - el.remove(); - } - } - return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? ""; -} - -function parseDocument(html: string): any { - const normalized = /<\s*html[\s>]/i.test(html) - ? html - : `${html}`; - return parseHTML(normalized).document; -} - -function sanitizeHtml(html: string): string { - const { document } = parseHTML(`
${html}
`); - const root = document.querySelector("#__root"); - if (!root) return html; - - for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) { - for (const el of root.querySelectorAll(selector)) { - el.remove(); - } - } - - return root.innerHTML; -} - -function flattenJsonLdItems(data: unknown): AnyRecord[] { - if (!data || typeof data !== "object") return []; - if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems); - - const item = data as AnyRecord; - if (Array.isArray(item["@graph"])) { - return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems); - } - - return [item]; -} - -function parseJsonLdScripts(document: any): AnyRecord[] { - const results: AnyRecord[] = []; - const scripts = document.querySelectorAll("script[type='application/ld+json']"); - - for (const script of scripts) { - try { - const data = JSON.parse(script.textContent ?? ""); - results.push(...flattenJsonLdItems(data)); - } catch { - // ignore malformed blocks - } - } - - return results; -} - -function isArticleType(item: AnyRecord): boolean { - const value = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"]; - return typeof value === "string" && ARTICLE_TYPES.has(value); -} - -function extractAuthorFromJsonLd(authorData: unknown): string | null { - if (typeof authorData === "string") return authorData; - if (!authorData || typeof authorData !== "object") return null; - - if (Array.isArray(authorData)) { - const names = authorData - .map((author) => extractAuthorFromJsonLd(author)) - .filter((name): name is string => Boolean(name)); - return names.length > 0 ? names.join(", ") : null; - } - - const author = authorData as AnyRecord; - return typeof author.name === "string" ? author.name : null; -} - -function parseJsonLdItem(item: AnyRecord): ExtractionCandidate | null { - if (!isArticleType(item)) return null; - - const rawContent = - (typeof item.articleBody === "string" && item.articleBody) || - (typeof item.text === "string" && item.text) || - (typeof item.description === "string" && item.description) || - null; - - if (!rawContent) return null; - - const content = rawContent.trim(); - const htmlLike = /<\/?[a-z][\s\S]*>/i.test(content); - const textContent = htmlLike ? extractTextFromHtml(content) : content; - - if (textContent.length < MIN_CONTENT_LENGTH) return null; - - return { - title: pickString(item.headline, item.name), - byline: extractAuthorFromJsonLd(item.author), - excerpt: pickString(item.description), - published: pickString(item.datePublished, item.dateCreated), - html: htmlLike ? content : null, - textContent, - method: "json-ld", + const metadata: PageMetadata = { + url, + title: result.title || "", + description: result.description || undefined, + author: result.author || undefined, + published: result.published || undefined, + captured_at: new Date().toISOString(), }; -} -function tryJsonLdExtraction(document: any): ExtractionCandidate | null { - for (const item of parseJsonLdScripts(document)) { - const extracted = parseJsonLdItem(item); - if (extracted) return extracted; - } - return null; -} - -function getByPath(value: unknown, path: string): unknown { - let current = value; - for (const part of path.split(".")) { - if (!current || typeof current !== "object") return undefined; - current = (current as AnyRecord)[part]; - } - return current; -} - -function isContentBlockArray(value: unknown): value is AnyRecord[] { - if (!Array.isArray(value) || value.length === 0) return false; - return value.slice(0, 5).some((item) => { - if (!item || typeof item !== "object") return false; - const obj = item as AnyRecord; - return "type" in obj || "text" in obj || "textHtml" in obj || "content" in obj; - }); -} - -function extractTextFromContentBlocks(blocks: AnyRecord[]): string { - const parts: string[] = []; - - function pushParagraph(text: string): void { - const trimmed = text.trim(); - if (!trimmed) return; - parts.push(trimmed, "\n\n"); - } - - function walk(node: unknown): void { - if (!node || typeof node !== "object") return; - const block = node as AnyRecord; - - if (typeof block.text === "string") { - pushParagraph(block.text); - return; - } - - if (typeof block.textHtml === "string") { - pushParagraph(extractTextFromHtml(block.textHtml)); - return; - } - - if (Array.isArray(block.items)) { - for (const item of block.items) { - if (item && typeof item === "object") { - const text = pickString((item as AnyRecord).text); - if (text) parts.push(`- ${text}\n`); - } - } - parts.push("\n"); - } - - if (Array.isArray(block.components)) { - for (const component of block.components) { - walk(component); - } - } - - if (Array.isArray(block.content)) { - for (const child of block.content) { - walk(child); - } - } - } - - for (const block of blocks) { - walk(block); - } - - return parts.join("").replace(/\n{3,}/g, "\n\n").trim(); -} - -function tryStringBodyExtraction( - content: string, - meta: AnyRecord, - document: any, - method: string -): ExtractionCandidate | null { - if (!content || content.length < MIN_CONTENT_LENGTH) return null; - - const isHtml = /<\/?[a-z][\s\S]*>/i.test(content); - const html = isHtml ? sanitizeHtml(content) : null; - const textContent = isHtml ? extractTextFromHtml(html) : content.trim(); - - if (textContent.length < MIN_CONTENT_LENGTH) return null; - - return { - title: pickString(meta.headline, meta.title, extractTitle(document)), - byline: pickString(meta.byline, meta.author), - excerpt: pickString(meta.description, meta.excerpt, generateExcerpt(null, textContent)), - published: pickString(meta.datePublished, meta.publishedAt, extractPublishedTime(document)), - html, - textContent, - method, - }; -} - -function tryNextDataExtraction(document: any): ExtractionCandidate | null { - try { - const script = document.querySelector("script#__NEXT_DATA__"); - if (!script?.textContent) return null; - - const data = JSON.parse(script.textContent) as AnyRecord; - const pageProps = (getByPath(data, "props.pageProps") ?? {}) as AnyRecord; - - for (const path of NEXT_DATA_CONTENT_PATHS) { - const value = getByPath(data, path); - - if (typeof value === "string") { - const parentPath = path.split(".").slice(0, -1).join("."); - const parent = (getByPath(data, parentPath) ?? {}) as AnyRecord; - const meta = { - ...pageProps, - ...parent, - title: parent.title ?? (pageProps.title as string | undefined), - }; - - const candidate = tryStringBodyExtraction(value, meta, document, "next-data"); - if (candidate) return candidate; - } - - if (isContentBlockArray(value)) { - const textContent = extractTextFromContentBlocks(value); - if (textContent.length < MIN_CONTENT_LENGTH) continue; - - return { - title: pickString( - getByPath(data, "props.pageProps.content.headline"), - getByPath(data, "props.pageProps.article.headline"), - getByPath(data, "props.pageProps.article.title"), - getByPath(data, "props.pageProps.post.title"), - pageProps.title, - extractTitle(document) - ), - byline: pickString( - getByPath(data, "props.pageProps.author.name"), - getByPath(data, "props.pageProps.article.author.name") - ), - excerpt: pickString( - getByPath(data, "props.pageProps.content.description"), - getByPath(data, "props.pageProps.article.description"), - pageProps.description, - generateExcerpt(null, textContent) - ), - published: pickString( - getByPath(data, "props.pageProps.content.datePublished"), - getByPath(data, "props.pageProps.article.datePublished"), - getByPath(data, "props.pageProps.publishedAt"), - extractPublishedTime(document) - ), - html: null, - textContent, - method: "next-data", - }; - } - } - } catch { - return null; - } - - return null; -} - -function buildReadabilityCandidate( - article: ReturnType, - document: any, - method: string -): ExtractionCandidate | null { - const textContent = article?.textContent?.trim() ?? ""; - if (textContent.length < MIN_CONTENT_LENGTH) return null; - - return { - title: pickString(article?.title, extractTitle(document)), - byline: pickString(article?.byline), - excerpt: pickString(article?.excerpt, generateExcerpt(null, textContent)), - published: pickString(article?.publishedTime, extractPublishedTime(document)), - html: article?.content ? sanitizeHtml(article.content) : null, - textContent, - method, - }; -} - -function tryReadability(document: any): ExtractionCandidate | null { - try { - const strictClone = document.cloneNode(true) as Document; - const strictResult = buildReadabilityCandidate( - new Readability(strictClone).parse(), - document, - "readability" - ); - if (strictResult) return strictResult; - - const relaxedClone = document.cloneNode(true) as Document; - return buildReadabilityCandidate( - new Readability(relaxedClone, { charThreshold: 120 }).parse(), - document, - "readability-relaxed" - ); - } catch { - return null; - } -} - -function trySelectorExtraction(document: any): ExtractionCandidate | null { - for (const selector of CONTENT_SELECTORS) { - const element = document.querySelector(selector); - if (!element) continue; - - const clone = element.cloneNode(true) as Element; - for (const removeSelector of REMOVE_SELECTORS) { - for (const node of clone.querySelectorAll(removeSelector)) { - node.remove(); - } - } - - const html = sanitizeHtml(clone.innerHTML); - const textContent = extractTextFromHtml(html); - if (textContent.length < MIN_CONTENT_LENGTH) continue; - - return { - title: extractTitle(document), - byline: null, - excerpt: generateExcerpt(null, textContent), - published: extractPublishedTime(document), - html, - textContent, - method: `selector:${selector}`, - }; - } - - return null; -} - -function tryBodyExtraction(document: any): ExtractionCandidate | null { - const body = document.body; - if (!body) return null; - - const clone = body.cloneNode(true) as Element; - for (const removeSelector of REMOVE_SELECTORS) { - for (const node of clone.querySelectorAll(removeSelector)) { - node.remove(); - } - } - - const html = sanitizeHtml(clone.innerHTML); - const textContent = extractTextFromHtml(html); - if (!textContent) return null; - - return { - title: extractTitle(document), - byline: null, - excerpt: generateExcerpt(null, textContent), - published: extractPublishedTime(document), - html, - textContent, - method: "body-fallback", - }; -} - -function pickBestCandidate(candidates: ExtractionCandidate[]): ExtractionCandidate | null { - if (candidates.length === 0) return null; - - const methodOrder = [ - "readability", - "readability-relaxed", - "next-data", - "json-ld", - "selector:", - "body-fallback", - ]; - - function methodRank(method: string): number { - const idx = methodOrder.findIndex((entry) => - entry.endsWith(":") ? method.startsWith(entry) : method === entry - ); - return idx === -1 ? methodOrder.length : idx; - } - - const ranked = [...candidates].sort((a, b) => { - const rankA = methodRank(a.method); - const rankB = methodRank(b.method); - if (rankA !== rankB) return rankA - rankB; - return (b.textContent.length ?? 0) - (a.textContent.length ?? 0); - }); - - for (const candidate of ranked) { - if (candidate.textContent.length >= GOOD_CONTENT_LENGTH) { - return candidate; - } - } - - for (const candidate of ranked) { - if (candidate.textContent.length >= MIN_CONTENT_LENGTH) { - return candidate; - } - } - - return ranked[0]; -} - -function extractFromHtml(html: string): ExtractionCandidate | null { - const document = parseDocument(html); - - const readabilityCandidate = tryReadability(document); - const nextDataCandidate = tryNextDataExtraction(document); - const jsonLdCandidate = tryJsonLdExtraction(document); - const selectorCandidate = trySelectorExtraction(document); - const bodyCandidate = tryBodyExtraction(document); - - const candidates = [ - readabilityCandidate, - nextDataCandidate, - jsonLdCandidate, - selectorCandidate, - bodyCandidate, - ].filter((candidate): candidate is ExtractionCandidate => Boolean(candidate)); - - const winner = pickBestCandidate(candidates); - if (!winner) return null; - - return { - ...winner, - title: winner.title ?? extractTitle(document), - published: winner.published ?? extractPublishedTime(document), - excerpt: winner.excerpt ?? generateExcerpt(null, winner.textContent), - }; -} - -const turndown = new TurndownService({ - headingStyle: "atx", - hr: "---", - bulletListMarker: "-", - codeBlockStyle: "fenced", - emDelimiter: "*", - strongDelimiter: "**", - linkStyle: "inlined", -}); - -turndown.use(gfm); - -turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]); - -turndown.addRule("collapseFigure", { - filter: "figure", - replacement(content) { - return `\n\n${content.trim()}\n\n`; - }, -}); - -turndown.addRule("dropInvisibleAnchors", { - filter(node) { - return node.nodeName === "A" && !(node as Element).textContent?.trim(); - }, - replacement() { - return ""; - }, -}); - -function normalizeMarkdown(markdown: string): string { - return markdown - .replace(/\r\n/g, "\n") - .replace(/[ \t]+\n/g, "\n") - .replace(/\n{3,}/g, "\n\n") - .trim(); -} - -function convertHtmlToMarkdown(html: string): string { - if (!html || !html.trim()) return ""; - - try { - const sanitized = sanitizeHtml(html); - return turndown.turndown(sanitized); - } catch { - return ""; - } -} - -function fallbackPlainText(html: string): string { - const document = parseDocument(html); - for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) { - for (const el of document.querySelectorAll(selector)) { - el.remove(); - } - } - const text = document.body?.textContent ?? document.documentElement?.textContent ?? ""; - return normalizeMarkdown(text.replace(/\s+/g, " ")); -} - -export function htmlToMarkdown(html: string): string { - if (!html || !html.trim()) return ""; - - const extracted = extractFromHtml(html); - if (!extracted) { - return fallbackPlainText(html); - } - - let markdown = extracted.html ? convertHtmlToMarkdown(extracted.html) : ""; - if (!markdown.trim()) { - markdown = extracted.textContent; - } - - return normalizeMarkdown(markdown); + return { metadata, markdown: result.content || "" }; } function escapeYamlValue(value: string): string { diff --git a/skills/baoyu-url-to-markdown/scripts/main.ts b/skills/baoyu-url-to-markdown/scripts/main.ts index 668fcc6..d9a84ab 100644 --- a/skills/baoyu-url-to-markdown/scripts/main.ts +++ b/skills/baoyu-url-to-markdown/scripts/main.ts @@ -4,7 +4,7 @@ import path from "node:path"; import process from "node:process"; import { CdpConnection, getFreePort, launchChrome, waitForChromeDebugPort, waitForNetworkIdle, waitForPageLoad, autoScroll, evaluateScript, killChrome } from "./cdp.js"; -import { cleanupAndExtractScript, htmlToMarkdown, createMarkdownDocument, type PageMetadata, type ConversionResult } from "./html-to-markdown.js"; +import { absolutizeUrlsScript, extractContent, createMarkdownDocument, type ConversionResult } from "./html-to-markdown.js"; import { resolveUrlToMarkdownDataDir } from "./paths.js"; import { DEFAULT_TIMEOUT_MS, CDP_CONNECT_TIMEOUT_MS, NETWORK_IDLE_TIMEOUT_MS, POST_LOAD_DELAY_MS, SCROLL_STEP_WAIT_MS, SCROLL_MAX_STEPS } from "./constants.js"; @@ -117,21 +117,11 @@ async function captureUrl(args: Args): Promise { } console.log("Capturing page content..."); - const extracted = await evaluateScript<{ title: string; description?: string; author?: string; published?: string; html: string }>( - cdp, sessionId, cleanupAndExtractScript, args.timeout + const { html } = await evaluateScript<{ html: string }>( + cdp, sessionId, absolutizeUrlsScript, args.timeout ); - const metadata: PageMetadata = { - url: args.url, - title: extracted.title || "", - description: extracted.description, - author: extracted.author, - published: extracted.published, - captured_at: new Date().toISOString() - }; - - const markdown = htmlToMarkdown(extracted.html); - return { metadata, markdown }; + return await extractContent(html, args.url); } finally { if (cdp) { try { await cdp.send("Browser.close", {}, { timeoutMs: 5_000 }); } catch {}