feat(baoyu-url-to-markdown): vendor baoyu-fetch runtime

2026-07-21 09:49:48 +08:00 · 2026-04-21 13:39:09 -05:00
parent 2f09f48726
commit d6442e1ab4
45 changed files with 7960 additions and 45 deletions
@@ -0,0 +1,74 @@
+import type { Adapter } from "../types";
+import { detectInteractionGate } from "../../browser/interaction-gates";
+import { captureNormalizedPageSnapshot } from "../../browser/page-snapshot";
+import { convertHtmlToMarkdown } from "../../extract/html-to-markdown";
+
+export const genericAdapter: Adapter = {
+  name: "generic",
+  match() {
+    return true;
+  },
+  async process(context) {
+    context.log.info(`Loading ${context.input.url.toString()} with generic adapter`);
+    await context.browser.goto(context.input.url.toString(), context.timeoutMs);
+
+    try {
+      await context.network.waitForIdle({
+        idleMs: 1_200,
+        timeoutMs: Math.min(context.timeoutMs, 15_000),
+      });
+    } catch {
+      context.log.debug("Network idle timed out on initial load; continuing.");
+    }
+
+    await context.browser.scrollToEnd({ maxSteps: 4, delayMs: 300 });
+
+    try {
+      await context.network.waitForIdle({
+        idleMs: 900,
+        timeoutMs: Math.min(context.timeoutMs, 10_000),
+      });
+    } catch {
+      context.log.debug("Network idle timed out after scrolling; continuing.");
+    }
+
+    const interaction = await detectInteractionGate(context.browser);
+    if (interaction) {
+      return {
+        status: "needs_interaction",
+        interaction,
+      };
+    }
+
+    const snapshot = await captureNormalizedPageSnapshot(context.browser);
+    const converted = await convertHtmlToMarkdown(snapshot.html, snapshot.finalUrl, {
+      enableRemoteMarkdownFallback: context.outputFormat === "markdown",
+      preserveBase64Images: context.downloadMedia,
+    });
+    const document = {
+      url: snapshot.finalUrl,
+      canonicalUrl: converted.metadata.canonicalUrl,
+      title: converted.metadata.title,
+      author: converted.metadata.author,
+      siteName: converted.metadata.siteName,
+      publishedAt: converted.metadata.publishedAt,
+      summary: converted.metadata.summary,
+      adapter: "generic",
+      metadata: {
+        coverImage: converted.metadata.coverImage,
+        language: converted.metadata.language,
+        capturedAt: converted.metadata.capturedAt,
+        conversionMethod: converted.conversionMethod,
+        fallbackReason: converted.fallbackReason,
+        kind: "generic/article",
+      },
+      content: converted.markdown ? [{ type: "markdown" as const, markdown: converted.markdown }] : [],
+    };
+
+    return {
+      status: "ok",
+      document,
+      media: converted.media,
+    };
+  },
+};