feat(baoyu-fetch): add URL reader CLI with Chrome CDP and site adapters

2026-08-07 09:23:04 +08:00 · 2026-03-27 14:11:00 -05:00
parent 2c14872e88
commit d0764c2739
65 changed files with 10235 additions and 0 deletions
@@ -0,0 +1,117 @@
+import type { Adapter, AdapterLoginInfo } from "../types";
+import { detectInteractionGate } from "../../browser/interaction-gates";
+import type { ExtractedDocument } from "../../extract/document";
+import { collectMediaFromDocument } from "../../media/markdown-media";
+import { extractArticleDocumentFromPayload } from "./article";
+import { buildNeedsLoginResult, detectXLogin } from "./login";
+import { extractStatusId, isXHost } from "./match";
+import { collectXJsonPayloads, waitForInitialXPayload } from "./payloads";
+import { extractSingleTweetDocumentFromPayload } from "./single";
+import { extractThreadDocumentFromPayloads } from "./thread";
+import { loadFullXThread } from "./thread-loader";
+
+function extractDocumentFromPayloads(
+  payloads: unknown[],
+  statusId: string,
+  pageUrl: string,
+): ExtractedDocument | null {
+  for (const payload of payloads) {
+    const articleDocument = extractArticleDocumentFromPayload(payload, statusId, pageUrl, payloads);
+    if (articleDocument) {
+      return articleDocument;
+    }
+  }
+
+  const threadDocument = extractThreadDocumentFromPayloads(payloads, statusId, pageUrl);
+  if (threadDocument) {
+    return threadDocument;
+  }
+
+  for (const payload of payloads) {
+    const singleDocument = extractSingleTweetDocumentFromPayload(payload, statusId, pageUrl);
+    if (singleDocument) {
+      return singleDocument;
+    }
+  }
+
+  return null;
+}
+
+async function ensureXLoginState(context: Parameters<Adapter["process"]>[0]): Promise<AdapterLoginInfo> {
+  return detectXLogin(context);
+}
+
+export const xAdapter: Adapter = {
+  name: "x",
+  match(input) {
+    return isXHost(input.url.hostname);
+  },
+  async checkLogin(context) {
+    return detectXLogin(context);
+  },
+  async process(context) {
+    const statusId = extractStatusId(context.input.url);
+    if (!statusId) {
+      return {
+        status: "no_document",
+      };
+    }
+
+    context.log.info(`Loading ${context.input.url.toString()} with x adapter`);
+    await context.browser.goto(context.input.url.toString(), context.timeoutMs);
+
+    const interaction = await detectInteractionGate(context.browser);
+    if (interaction) {
+      return {
+        status: "needs_interaction",
+        interaction,
+      };
+    }
+
+    let login = await ensureXLoginState(context);
+    if (login.state === "logged_out") {
+      return buildNeedsLoginResult(login);
+    }
+
+    await waitForInitialXPayload(context);
+    await loadFullXThread(context, statusId);
+
+    const pageUrl = await context.browser.getURL();
+    const postLoadInteraction = await detectInteractionGate(context.browser);
+    if (postLoadInteraction) {
+      return {
+        status: "needs_interaction",
+        interaction: postLoadInteraction,
+        login,
+      };
+    }
+
+    login = await ensureXLoginState(context).catch(() => login);
+    if (login.state === "logged_out") {
+      return buildNeedsLoginResult(login);
+    }
+
+    const payloads = await collectXJsonPayloads(context);
+    if (payloads.length === 0) {
+      return {
+        status: "no_document",
+        login,
+      };
+    }
+
+    const document = extractDocumentFromPayloads(payloads, statusId, pageUrl);
+    if (document) {
+      return {
+        status: "ok",
+        document,
+        media: collectMediaFromDocument(document),
+        login,
+      };
+    }
+
+    return {
+      status: "no_document",
+      login,
+    };
+  },
+};