baoyu-skills/packages/baoyu-fetch/src/extract/html-extractor.ts

import { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import type { ExtractedDocument } from "./document";

function getMetaContent(document: Document, selectors: string[]): string | undefined {
  for (const selector of selectors) {
    const value = document.querySelector(selector)?.getAttribute("content")?.trim();
    if (value) {
      return value;
    }
  }
  return undefined;
}

export function extractDocumentFromHtml(input: {
  url: string;
  html: string;
  adapter?: string;
}): ExtractedDocument {
  const dom = new JSDOM(input.html, { url: input.url });
  const document = dom.window.document;

  const canonicalUrl =
    document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
    getMetaContent(document, ['meta[property="og:url"]']);

  const siteName = getMetaContent(document, [
    'meta[property="og:site_name"]',
    'meta[name="application-name"]',
  ]);

  const metadataAuthor = getMetaContent(document, [
    'meta[name="author"]',
    'meta[property="article:author"]',
    'meta[name="twitter:creator"]',
  ]);

  const publishedAt = getMetaContent(document, [
    'meta[property="article:published_time"]',
    'meta[name="pubdate"]',
    'meta[name="date"]',
    'meta[itemprop="datePublished"]',
  ]);

  const article = new Readability(document).parse();
  const title =
    article?.title?.trim() ||
    getMetaContent(document, ['meta[property="og:title"]']) ||
    document.title.trim() ||
    undefined;

  const summary =
    article?.excerpt?.trim() ||
    getMetaContent(document, [
      'meta[name="description"]',
      'meta[property="og:description"]',
      'meta[name="twitter:description"]',
    ]);

  const contentHtml =
    article?.content?.trim() ||
    document.querySelector("main")?.innerHTML?.trim() ||
    document.body?.innerHTML?.trim() ||
    "";

  const author = article?.byline?.trim() || metadataAuthor;

  return {
    url: input.url,
    canonicalUrl,
    title,
    author,
    siteName,
    publishedAt,
    summary,
    adapter: input.adapter ?? "generic",
    metadata: {
      language: document.documentElement.lang || undefined,
    },
    content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
  };
}