baoyu-skills/skills/baoyu-url-to-markdown/scripts/lib/adapters/generic/index.ts

import type { Adapter } from "../types";
import { detectInteractionGate } from "../../browser/interaction-gates";
import { captureNormalizedPageSnapshot } from "../../browser/page-snapshot";
import { convertHtmlToMarkdown } from "../../extract/html-to-markdown";

export const genericAdapter: Adapter = {
  name: "generic",
  match() {
    return true;
  },
  async process(context) {
    context.log.info(`Loading ${context.input.url.toString()} with generic adapter`);
    await context.browser.goto(context.input.url.toString(), context.timeoutMs);

    try {
      await context.network.waitForIdle({
        idleMs: 1_200,
        timeoutMs: Math.min(context.timeoutMs, 15_000),
      });
    } catch {
      context.log.debug("Network idle timed out on initial load; continuing.");
    }

    await context.browser.scrollToEnd({ maxSteps: 4, delayMs: 300 });

    try {
      await context.network.waitForIdle({
        idleMs: 900,
        timeoutMs: Math.min(context.timeoutMs, 10_000),
      });
    } catch {
      context.log.debug("Network idle timed out after scrolling; continuing.");
    }

    const interaction = await detectInteractionGate(context.browser);
    if (interaction) {
      return {
        status: "needs_interaction",
        interaction,
      };
    }

    const snapshot = await captureNormalizedPageSnapshot(context.browser);
    const converted = await convertHtmlToMarkdown(snapshot.html, snapshot.finalUrl, {
      enableRemoteMarkdownFallback: context.outputFormat === "markdown",
      preserveBase64Images: context.downloadMedia,
    });
    const document = {
      url: snapshot.finalUrl,
      canonicalUrl: converted.metadata.canonicalUrl,
      title: converted.metadata.title,
      author: converted.metadata.author,
      siteName: converted.metadata.siteName,
      publishedAt: converted.metadata.publishedAt,
      summary: converted.metadata.summary,
      adapter: "generic",
      metadata: {
        coverImage: converted.metadata.coverImage,
        language: converted.metadata.language,
        capturedAt: converted.metadata.capturedAt,
        conversionMethod: converted.conversionMethod,
        fallbackReason: converted.fallbackReason,
        kind: "generic/article",
      },
      content: converted.markdown ? [{ type: "markdown" as const, markdown: converted.markdown }] : [],
    };

    return {
      status: "ok",
      document,
      media: converted.media,
    };
  },
};