feat(baoyu-fetch): add URL reader CLI with Chrome CDP and site adapters

2026-07-22 10:19:46 +08:00 · 2026-03-27 14:11:00 -05:00
parent 2c14872e88
commit d0764c2739
65 changed files with 10235 additions and 0 deletions
@@ -0,0 +1,83 @@
+import { Readability } from "@mozilla/readability";
+import { JSDOM } from "jsdom";
+import type { ExtractedDocument } from "./document";
+
+function getMetaContent(document: Document, selectors: string[]): string | undefined {
+  for (const selector of selectors) {
+    const value = document.querySelector(selector)?.getAttribute("content")?.trim();
+    if (value) {
+      return value;
+    }
+  }
+  return undefined;
+}
+
+export function extractDocumentFromHtml(input: {
+  url: string;
+  html: string;
+  adapter?: string;
+}): ExtractedDocument {
+  const dom = new JSDOM(input.html, { url: input.url });
+  const document = dom.window.document;
+
+  const canonicalUrl =
+    document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
+    getMetaContent(document, ['meta[property="og:url"]']);
+
+  const siteName = getMetaContent(document, [
+    'meta[property="og:site_name"]',
+    'meta[name="application-name"]',
+  ]);
+
+  const metadataAuthor = getMetaContent(document, [
+    'meta[name="author"]',
+    'meta[property="article:author"]',
+    'meta[name="twitter:creator"]',
+  ]);
+
+  const publishedAt = getMetaContent(document, [
+    'meta[property="article:published_time"]',
+    'meta[name="pubdate"]',
+    'meta[name="date"]',
+    'meta[itemprop="datePublished"]',
+  ]);
+
+  const article = new Readability(document).parse();
+  const title =
+    article?.title?.trim() ||
+    getMetaContent(document, ['meta[property="og:title"]']) ||
+    document.title.trim() ||
+    undefined;
+
+  const summary =
+    article?.excerpt?.trim() ||
+    getMetaContent(document, [
+      'meta[name="description"]',
+      'meta[property="og:description"]',
+      'meta[name="twitter:description"]',
+    ]);
+
+  const contentHtml =
+    article?.content?.trim() ||
+    document.querySelector("main")?.innerHTML?.trim() ||
+    document.body?.innerHTML?.trim() ||
+    "";
+
+  const author = article?.byline?.trim() || metadataAuthor;
+
+  return {
+    url: input.url,
+    canonicalUrl,
+    title,
+    author,
+    siteName,
+    publishedAt,
+    summary,
+    adapter: input.adapter ?? "generic",
+    metadata: {
+      language: document.documentElement.lang || undefined,
+    },
+    content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
+  };
+}
+