feat(baoyu-fetch): add URL reader CLI with Chrome CDP and site adapters

packages/baoyu-fetch/src/browser/cdp-client.ts

@@ -0,0 +1,258 @@
+import { EventEmitter } from "node:events";
+import WebSocket from "ws";
+
+type JsonObject = Record<string, unknown>;
+
+interface CdpPendingCommand {
+  resolve(value: unknown): void;
+  reject(error: unknown): void;
+  method: string;
+}
+
+interface CdpErrorShape {
+  message?: string;
+}
+
+interface CdpCommandResult<T> {
+  result?: T;
+  error?: CdpErrorShape;
+}
+
+interface CreatePageSessionOptions {
+  initialUrl?: string;
+  visible?: boolean;
+}
+
+export class TargetSession extends EventEmitter {
+  constructor(
+    private readonly client: CdpClient,
+    public readonly targetId: string,
+    public readonly sessionId: string,
+  ) {
+    super();
+  }
+
+  async send<T>(method: string, params: JsonObject = {}): Promise<T> {
+    return this.client.sendSessionCommand<T>(this.sessionId, method, params);
+  }
+
+  handleEvent(method: string, params: JsonObject): void {
+    this.emit(method, params);
+    this.emit("event", { method, params });
+  }
+
+  async waitForEvent<T extends JsonObject>(
+    method: string,
+    predicate?: (params: T) => boolean,
+    timeoutMs = 30_000,
+  ): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this.off(method, listener);
+        reject(new Error(`Timed out waiting for ${method}`));
+      }, timeoutMs);
+
+      const listener = (params: T): void => {
+        if (predicate && !predicate(params)) {
+          return;
+        }
+        clearTimeout(timeout);
+        this.off(method, listener);
+        resolve(params);
+      };
+
+      this.on(method, listener);
+    });
+  }
+}
+
+export class CdpClient {
+  private readonly ws: WebSocket;
+  private readonly pending = new Map<number, CdpPendingCommand>();
+  private readonly sessions = new Map<string, TargetSession>();
+  private nextId = 1;
+
+  private constructor(ws: WebSocket) {
+    this.ws = ws;
+    this.ws.on("message", (raw) => {
+      this.handleMessage(raw.toString());
+    });
+  }
+
+  static async connect(browserWsUrl: string): Promise<CdpClient> {
+    const ws = await new Promise<WebSocket>((resolve, reject) => {
+      const socket = new WebSocket(browserWsUrl);
+      socket.once("open", () => resolve(socket));
+      socket.once("error", (error) => reject(error));
+    });
+
+    return new CdpClient(ws);
+  }
+
+  private handleMessage(rawMessage: string): void {
+    const message = JSON.parse(rawMessage) as {
+      id?: number;
+      sessionId?: string;
+      method?: string;
+      params?: JsonObject;
+      result?: unknown;
+      error?: CdpErrorShape;
+    };
+
+    if (typeof message.id === "number") {
+      const pending = this.pending.get(message.id);
+      if (!pending) {
+        return;
+      }
+      this.pending.delete(message.id);
+      if (message.error) {
+        pending.reject(new Error(`${pending.method}: ${message.error.message ?? "Unknown CDP error"}`));
+        return;
+      }
+      pending.resolve(message.result);
+      return;
+    }
+
+    if (typeof message.sessionId === "string" && typeof message.method === "string") {
+      const session = this.sessions.get(message.sessionId);
+      if (session) {
+        session.handleEvent(message.method, (message.params ?? {}) as JsonObject);
+      }
+    }
+  }
+
+  private async sendCommand<T>(
+    method: string,
+    params: JsonObject = {},
+    sessionId?: string,
+  ): Promise<T> {
+    const id = this.nextId;
+    this.nextId += 1;
+
+    const payload = sessionId ? { id, method, params, sessionId } : { id, method, params };
+
+    const result = new Promise<T>((resolve, reject) => {
+      this.pending.set(id, {
+        resolve: (value) => resolve(value as T),
+        reject,
+        method,
+      });
+    });
+
+    this.ws.send(JSON.stringify(payload));
+    return result;
+  }
+
+  async sendBrowserCommand<T>(method: string, params: JsonObject = {}): Promise<T> {
+    return this.sendCommand<T>(method, params);
+  }
+
+  async sendSessionCommand<T>(sessionId: string, method: string, params: JsonObject = {}): Promise<T> {
+    return this.sendCommand<T>(method, params, sessionId);
+  }
+
+  private async createPageTarget(initialUrl: string, visible = false): Promise<{ targetId: string }> {
+    const attempts: JsonObject[] = visible
+      ? [
+          {
+            url: initialUrl,
+            newWindow: true,
+            focus: true,
+          },
+          {
+            url: initialUrl,
+            focus: true,
+          },
+          {
+            url: initialUrl,
+          },
+        ]
+      : [
+          {
+            url: initialUrl,
+            hidden: true,
+          },
+          {
+            url: initialUrl,
+            background: true,
+            focus: false,
+          },
+          {
+            url: initialUrl,
+          },
+        ];
+
+    let lastError: unknown;
+
+    for (const params of attempts) {
+      try {
+        return await this.sendBrowserCommand<{ targetId: string }>("Target.createTarget", params);
+      } catch (error) {
+        lastError = error;
+      }
+    }
+
+    throw lastError instanceof Error ? lastError : new Error("Target.createTarget failed");
+  }
+
+  async createPageSession(options: CreatePageSessionOptions = {}): Promise<TargetSession> {
+    const initialUrl = options.initialUrl ?? "about:blank";
+    const created = await this.createPageTarget(initialUrl, Boolean(options.visible));
+    const attached = await this.sendBrowserCommand<{ sessionId: string }>("Target.attachToTarget", {
+      targetId: created.targetId,
+      flatten: true,
+    });
+
+    const session = new TargetSession(this, created.targetId, attached.sessionId);
+    this.sessions.set(attached.sessionId, session);
+
+    if (options.visible) {
+      await this.sendBrowserCommand("Target.activateTarget", {
+        targetId: created.targetId,
+      }).catch(() => {});
+    }
+
+    await session.send("Page.enable");
+    await session.send("Runtime.enable");
+    await session.send("DOM.enable");
+
+    if (options.visible) {
+      await session.send("Page.bringToFront").catch(() => {});
+    }
+
+    return session;
+  }
+
+  async closeTarget(targetId: string): Promise<void> {
+    try {
+      await this.sendBrowserCommand("Target.closeTarget", { targetId });
+    } catch {
+      // Target may already be gone.
+    }
+  }
+
+  async close(): Promise<void> {
+    await new Promise<void>((resolve) => {
+      if (this.ws.readyState === WebSocket.CLOSED) {
+        resolve();
+        return;
+      }
+      this.ws.once("close", () => resolve());
+      this.ws.close();
+    });
+  }
+}
+
+export async function evaluateRuntime<T>(session: TargetSession, expression: string): Promise<T> {
+  const response = await session.send<CdpCommandResult<{ value?: T; description?: string }>>("Runtime.evaluate", {
+    expression,
+    awaitPromise: true,
+    returnByValue: true,
+  });
+
+  if (response.error) {
+    throw new Error(response.error.message ?? "Runtime.evaluate failed");
+  }
+
+  return (response.result?.value as T | undefined) ?? (undefined as T);
+}

packages/baoyu-fetch/src/browser/chrome-launcher.ts

@@ -0,0 +1,117 @@
+import { launch, type LaunchedChrome } from "chrome-launcher";
+import type { Logger } from "../utils/logger";
+import { ensureChromeProfileDir, findExistingChromeDebugPort, resolveChromeProfileDir } from "./profile";
+
+interface ChromeVersionResponse {
+  webSocketDebuggerUrl: string;
+}
+
+export interface ChromeConnectOptions {
+  cdpUrl?: string;
+  browserPath?: string;
+  headless?: boolean;
+  logger?: Logger;
+  profileDir?: string;
+}
+
+export interface ChromeConnection {
+  browserWsUrl: string;
+  origin?: string;
+  port?: number;
+  profileDir?: string;
+  launched: boolean;
+  close(): Promise<void>;
+}
+
+async function fetchJson<T>(url: string): Promise<T> {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function connectToHttpEndpoint(origin: string): Promise<ChromeConnection> {
+  const normalizedOrigin = origin.replace(/\/$/, "");
+  const version = await fetchJson<ChromeVersionResponse>(`${normalizedOrigin}/json/version`);
+  return {
+    browserWsUrl: version.webSocketDebuggerUrl,
+    origin: normalizedOrigin,
+    port: Number(new URL(normalizedOrigin).port || 80),
+    launched: false,
+    async close() {
+      // Reused external Chrome, nothing to close here.
+    },
+  };
+}
+
+async function tryReuseChrome(profileDir: string, logger?: Logger): Promise<ChromeConnection | null> {
+  const port = await findExistingChromeDebugPort({ profileDir });
+  if (!port) {
+    return null;
+  }
+
+  const origin = `http://127.0.0.1:${port}`;
+  try {
+    const connection = await connectToHttpEndpoint(origin);
+    logger?.info(`Reusing Chrome debugger at ${origin} for profile ${profileDir}`);
+    return {
+      ...connection,
+      profileDir,
+    };
+  } catch {
+    // Debugger disappeared between detection and connect.
+  }
+  return null;
+}
+
+export async function connectChrome(options: ChromeConnectOptions): Promise<ChromeConnection> {
+  if (options.cdpUrl) {
+    if (options.cdpUrl.startsWith("ws://") || options.cdpUrl.startsWith("wss://")) {
+      return {
+        browserWsUrl: options.cdpUrl,
+        launched: false,
+        async close() {},
+      };
+    }
+    return connectToHttpEndpoint(options.cdpUrl);
+  }
+
+  const profileDir = ensureChromeProfileDir(resolveChromeProfileDir(options.profileDir));
+  const reused = await tryReuseChrome(profileDir, options.logger);
+  if (reused) {
+    return reused;
+  }
+
+  options.logger?.warn(`No running Chrome debugger found for profile ${profileDir}. Launching Chrome with that profile.`);
+
+  const launchedChrome: LaunchedChrome = await launch({
+    chromePath: options.browserPath,
+    userDataDir: profileDir,
+    chromeFlags: [
+      "--disable-background-networking",
+      "--disable-default-apps",
+      "--disable-popup-blocking",
+      "--disable-sync",
+      "--no-first-run",
+      "--no-default-browser-check",
+      "--remote-allow-origins=*",
+      ...(!options.headless ? ["--no-startup-window"] : []),
+      ...(options.headless ? ["--headless=new"] : []),
+    ],
+  });
+
+  const origin = `http://127.0.0.1:${launchedChrome.port}`;
+  const version = await fetchJson<ChromeVersionResponse>(`${origin}/json/version`);
+
+  return {
+    browserWsUrl: version.webSocketDebuggerUrl,
+    origin,
+    port: launchedChrome.port,
+    profileDir,
+    launched: true,
+    async close() {
+      launchedChrome.kill();
+    },
+  };
+}

packages/baoyu-fetch/src/browser/interaction-gates.ts

@@ -0,0 +1,123 @@
+import type { WaitForInteractionRequest } from "../adapters/types";
+import type { BrowserSession } from "./session";
+
+interface GateSnapshot {
+  title: string;
+  currentUrl: string;
+  bodyText: string;
+  hasCloudflareTurnstile: boolean;
+  hasCloudflareChallenge: boolean;
+  hasRecaptcha: boolean;
+  hasRecaptchaIframe: boolean;
+  hasHcaptcha: boolean;
+  hasHcaptchaIframe: boolean;
+}
+
+export function detectInteractionGateFromSnapshot(snapshot: GateSnapshot): WaitForInteractionRequest | null {
+  const text = snapshot.bodyText.toLowerCase();
+  const title = snapshot.title.toLowerCase();
+  const url = snapshot.currentUrl.toLowerCase();
+
+  if (
+    snapshot.hasCloudflareTurnstile ||
+    snapshot.hasCloudflareChallenge ||
+    title.includes("just a moment") ||
+    text.includes("verify you are human") ||
+    text.includes("checking your browser before accessing") ||
+    text.includes("enable javascript and cookies to continue") ||
+    url.includes("/cdn-cgi/challenge-platform/")
+  ) {
+    return {
+      type: "wait_for_interaction",
+      kind: "cloudflare",
+      provider: "cloudflare",
+      reason: "Cloudflare human verification detected",
+      prompt: "Please complete the Cloudflare verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
+      requiresVisibleBrowser: true,
+    };
+  }
+
+  if (
+    snapshot.hasRecaptcha ||
+    snapshot.hasRecaptchaIframe ||
+    text.includes("i'm not a robot") ||
+    text.includes("recaptcha")
+  ) {
+    return {
+      type: "wait_for_interaction",
+      kind: "recaptcha",
+      provider: "google_recaptcha",
+      reason: "Google reCAPTCHA detected",
+      prompt: "Please complete the reCAPTCHA verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
+      requiresVisibleBrowser: true,
+    };
+  }
+
+  if (
+    snapshot.hasHcaptcha ||
+    snapshot.hasHcaptchaIframe ||
+    text.includes("hcaptcha")
+  ) {
+    return {
+      type: "wait_for_interaction",
+      kind: "hcaptcha",
+      provider: "hcaptcha",
+      reason: "hCaptcha verification detected",
+      prompt: "Please complete the hCaptcha verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
+      requiresVisibleBrowser: true,
+    };
+  }
+
+  return null;
+}
+
+export async function detectInteractionGate(browser: BrowserSession): Promise<WaitForInteractionRequest | null> {
+  const snapshot = await browser.evaluate<GateSnapshot>(`
+    (() => {
+      const bodyText = (document.body?.innerText ?? "").slice(0, 4000);
+      return {
+        title: document.title ?? "",
+        currentUrl: window.location.href,
+        bodyText,
+        hasCloudflareTurnstile: Boolean(
+          document.querySelector(
+            '.cf-turnstile, [name="cf-turnstile-response"], iframe[src*="challenges.cloudflare.com"]'
+          )
+        ),
+        hasCloudflareChallenge: Boolean(
+          document.querySelector(
+            '#challenge-running, #cf-challenge-running, .challenge-platform, [data-ray], [data-translate="checking_browser"]'
+          )
+        ),
+        hasRecaptcha: Boolean(
+          document.querySelector(
+            '.g-recaptcha, textarea[name="g-recaptcha-response"], iframe[title*="reCAPTCHA"]'
+          )
+        ),
+        hasRecaptchaIframe: Boolean(
+          document.querySelector('iframe[src*="google.com/recaptcha"], iframe[src*="recaptcha/api2"]')
+        ),
+        hasHcaptcha: Boolean(
+          document.querySelector(
+            '.h-captcha, textarea[name="h-captcha-response"], iframe[title*="hCaptcha"]'
+          )
+        ),
+        hasHcaptchaIframe: Boolean(
+          document.querySelector('iframe[src*="hcaptcha.com"]')
+        ),
+      };
+    })()
+  `).catch(() => ({
+    title: "",
+    currentUrl: "",
+    bodyText: "",
+    hasCloudflareTurnstile: false,
+    hasCloudflareChallenge: false,
+    hasRecaptcha: false,
+    hasRecaptchaIframe: false,
+    hasHcaptcha: false,
+    hasHcaptchaIframe: false,
+  }));
+
+  return detectInteractionGateFromSnapshot(snapshot);
+}

packages/baoyu-fetch/src/browser/network-journal.ts

@@ -0,0 +1,235 @@
+import type { TargetSession } from "./cdp-client";
+import type { Logger } from "../utils/logger";
+
+type JsonObject = Record<string, unknown>;
+
+export interface NetworkEntry {
+  requestId: string;
+  url: string;
+  method: string;
+  resourceType: string;
+  timestamp: number;
+  requestHeaders?: Record<string, string>;
+  requestBody?: string;
+  status?: number;
+  statusText?: string;
+  responseHeaders?: Record<string, string>;
+  mimeType?: string;
+  body?: string;
+  bodyBase64?: boolean;
+  bodyError?: string;
+  failed?: boolean;
+  failureReason?: string;
+  finished: boolean;
+}
+
+function normalizeHeaders(headers: unknown): Record<string, string> | undefined {
+  if (!headers || typeof headers !== "object") {
+    return undefined;
+  }
+  return Object.fromEntries(
+    Object.entries(headers as Record<string, unknown>).map(([key, value]) => [key, String(value)]),
+  );
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export class NetworkJournal {
+  private readonly entries = new Map<string, NetworkEntry>();
+  private lastActivityAt = Date.now();
+  private started = false;
+
+  constructor(
+    private readonly session: TargetSession,
+    private readonly log: Logger,
+  ) {}
+
+  async start(): Promise<void> {
+    if (this.started) {
+      return;
+    }
+
+    this.started = true;
+    this.session.on("Network.requestWillBeSent", this.handleRequestWillBeSent);
+    this.session.on("Network.responseReceived", this.handleResponseReceived);
+    this.session.on("Network.loadingFinished", this.handleLoadingFinished);
+    this.session.on("Network.loadingFailed", this.handleLoadingFailed);
+    await this.session.send("Network.enable");
+  }
+
+  stop(): void {
+    if (!this.started) {
+      return;
+    }
+    this.session.off("Network.requestWillBeSent", this.handleRequestWillBeSent);
+    this.session.off("Network.responseReceived", this.handleResponseReceived);
+    this.session.off("Network.loadingFinished", this.handleLoadingFinished);
+    this.session.off("Network.loadingFailed", this.handleLoadingFailed);
+    this.started = false;
+  }
+
+  private touch(): void {
+    this.lastActivityAt = Date.now();
+  }
+
+  private readonly handleRequestWillBeSent = (params: JsonObject): void => {
+    const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+    const request = params.request as JsonObject | undefined;
+    if (!requestId || !request) {
+      return;
+    }
+
+    this.touch();
+    this.entries.set(requestId, {
+      requestId,
+      url: String(request.url ?? ""),
+      method: String(request.method ?? "GET"),
+      resourceType: String(params.type ?? "Other"),
+      timestamp: Date.now(),
+      requestHeaders: normalizeHeaders(request.headers),
+      requestBody: typeof request.postData === "string" ? request.postData : undefined,
+      finished: false,
+    });
+  };
+
+  private readonly handleResponseReceived = (params: JsonObject): void => {
+    const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+    const response = params.response as JsonObject | undefined;
+    if (!requestId || !response) {
+      return;
+    }
+
+    this.touch();
+    const existing = this.entries.get(requestId);
+    if (!existing) {
+      return;
+    }
+
+    existing.status = typeof response.status === "number" ? response.status : undefined;
+    existing.statusText = typeof response.statusText === "string" ? response.statusText : undefined;
+    existing.responseHeaders = normalizeHeaders(response.headers);
+    existing.mimeType = typeof response.mimeType === "string" ? response.mimeType : undefined;
+    this.entries.set(requestId, existing);
+  };
+
+  private readonly handleLoadingFinished = (params: JsonObject): void => {
+    const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+    if (!requestId) {
+      return;
+    }
+
+    this.touch();
+    const existing = this.entries.get(requestId);
+    if (!existing) {
+      return;
+    }
+    existing.finished = true;
+    this.entries.set(requestId, existing);
+  };
+
+  private readonly handleLoadingFailed = (params: JsonObject): void => {
+    const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+    if (!requestId) {
+      return;
+    }
+
+    this.touch();
+    const existing = this.entries.get(requestId);
+    if (!existing) {
+      return;
+    }
+    existing.finished = true;
+    existing.failed = true;
+    existing.failureReason = typeof params.errorText === "string" ? params.errorText : "Unknown error";
+    this.entries.set(requestId, existing);
+  };
+
+  getEntries(): NetworkEntry[] {
+    return Array.from(this.entries.values());
+  }
+
+  findEntries(predicate: (entry: NetworkEntry) => boolean): NetworkEntry[] {
+    return this.getEntries().filter(predicate);
+  }
+
+  async waitForIdle(options: { idleMs?: number; timeoutMs?: number } = {}): Promise<void> {
+    const idleMs = options.idleMs ?? 1_200;
+    const timeoutMs = options.timeoutMs ?? 15_000;
+    const startedAt = Date.now();
+
+    while (Date.now() - startedAt < timeoutMs) {
+      if (Date.now() - this.lastActivityAt >= idleMs) {
+        return;
+      }
+      await sleep(Math.min(150, idleMs));
+    }
+
+    throw new Error("Timed out waiting for network idle");
+  }
+
+  async waitForResponse(
+    predicate: (entry: NetworkEntry) => boolean,
+    options: { timeoutMs?: number } = {},
+  ): Promise<NetworkEntry> {
+    const timeoutMs = options.timeoutMs ?? 10_000;
+    const startedAt = Date.now();
+
+    while (Date.now() - startedAt < timeoutMs) {
+      const matched = this.getEntries().find((entry) => entry.finished && predicate(entry));
+      if (matched) {
+        return matched;
+      }
+      await sleep(150);
+    }
+
+    throw new Error("Timed out waiting for matching network response");
+  }
+
+  async ensureBody(entry: NetworkEntry): Promise<string | undefined> {
+    if (entry.body !== undefined) {
+      return entry.body;
+    }
+    if (entry.bodyError || entry.failed || !entry.finished) {
+      return undefined;
+    }
+
+    try {
+      const result = await this.session.send<{ body: string; base64Encoded: boolean }>("Network.getResponseBody", {
+        requestId: entry.requestId,
+      });
+      entry.bodyBase64 = result.base64Encoded;
+      entry.body = result.base64Encoded ? Buffer.from(result.body, "base64").toString("utf8") : result.body;
+      return entry.body;
+    } catch (error) {
+      entry.bodyError = error instanceof Error ? error.message : String(error);
+      this.log.debug(`Failed to fetch response body for ${entry.url}: ${entry.bodyError}`);
+      return undefined;
+    }
+  }
+
+  async getJsonBody(entry: NetworkEntry): Promise<unknown | null> {
+    const body = await this.ensureBody(entry);
+    if (!body) {
+      return null;
+    }
+
+    try {
+      return JSON.parse(body);
+    } catch {
+      return null;
+    }
+  }
+
+  async toJSON(options: { includeBodies?: boolean } = {}): Promise<NetworkEntry[]> {
+    const entries = this.getEntries();
+    if (!options.includeBodies) {
+      return entries;
+    }
+
+    await Promise.all(entries.map((entry) => this.ensureBody(entry)));
+    return entries;
+  }
+}
+

packages/baoyu-fetch/src/browser/page-snapshot.ts

@@ -0,0 +1,105 @@
+import type { BrowserSession } from "./session";
+
+export interface CapturedPageSnapshot {
+  html: string;
+  finalUrl: string;
+}
+
+export const CAPTURE_NORMALIZED_PAGE_SCRIPT = String.raw`
+(() => {
+  const baseUrl = document.baseURI || location.href;
+  const htmlClone = document.documentElement.cloneNode(true);
+
+  function materializeShadowDom(sourceRoot, cloneRoot) {
+    const sourceElements = Array.from(sourceRoot.querySelectorAll("*"));
+    const cloneElements = Array.from(cloneRoot.querySelectorAll("*"));
+
+    for (let index = sourceElements.length - 1; index >= 0; index -= 1) {
+      const sourceElement = sourceElements[index];
+      const cloneElement = cloneElements[index];
+      const shadowRoot = sourceElement && sourceElement.shadowRoot;
+      if (!shadowRoot || !cloneElement || !shadowRoot.innerHTML) {
+        continue;
+      }
+
+      if (cloneElement.tagName && cloneElement.tagName.includes("-")) {
+        const wrapper = document.createElement("div");
+        wrapper.setAttribute("data-shadow-host", cloneElement.tagName.toLowerCase());
+        wrapper.innerHTML = shadowRoot.innerHTML;
+        cloneElement.replaceWith(wrapper);
+      } else {
+        cloneElement.innerHTML = shadowRoot.innerHTML;
+      }
+    }
+  }
+
+  function toAbsolute(url) {
+    if (!url) return url;
+    try {
+      return new URL(url, baseUrl).href;
+    } catch {
+      return url;
+    }
+  }
+
+  function absolutizeAttribute(root, selector, attribute) {
+    root.querySelectorAll(selector).forEach((element) => {
+      const value = element.getAttribute(attribute);
+      if (!value) return;
+      const absolute = toAbsolute(value);
+      if (absolute) {
+        element.setAttribute(attribute, absolute);
+      }
+    });
+  }
+
+  function absolutizeSrcset(root, selector) {
+    root.querySelectorAll(selector).forEach((element) => {
+      const srcset = element.getAttribute("srcset");
+      if (!srcset) return;
+      element.setAttribute(
+        "srcset",
+        srcset
+          .split(",")
+          .map((part) => {
+            const trimmed = part.trim();
+            if (!trimmed) return "";
+            const [url, ...descriptor] = trimmed.split(/\s+/);
+            const absolute = toAbsolute(url);
+            return descriptor.length > 0 ? absolute + " " + descriptor.join(" ") : absolute;
+          })
+          .filter(Boolean)
+          .join(", "),
+      );
+    });
+  }
+
+  materializeShadowDom(document.documentElement, htmlClone);
+
+  htmlClone
+    .querySelectorAll("img[data-src], video[data-src], audio[data-src], source[data-src]")
+    .forEach((element) => {
+      const dataSource = element.getAttribute("data-src");
+      const current = element.getAttribute("src");
+      if (dataSource && (!current || current === "" || current.startsWith("data:"))) {
+        element.setAttribute("src", dataSource);
+      }
+    });
+
+  absolutizeAttribute(htmlClone, "a[href]", "href");
+  absolutizeAttribute(htmlClone, "img[src], video[src], audio[src], source[src], iframe[src]", "src");
+  absolutizeAttribute(htmlClone, "video[poster]", "poster");
+  absolutizeSrcset(htmlClone, "img[srcset], source[srcset]");
+
+  return {
+    html: "<!doctype html>\n" + htmlClone.outerHTML,
+    finalUrl: location.href,
+  };
+})()
+`;
+
+export async function captureNormalizedPageSnapshot(
+  browser: BrowserSession,
+): Promise<CapturedPageSnapshot> {
+  return browser.evaluate<CapturedPageSnapshot>(CAPTURE_NORMALIZED_PAGE_SCRIPT);
+}

packages/baoyu-fetch/src/browser/profile.ts

@@ -0,0 +1,148 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import process from "node:process";
+import { spawnSync } from "node:child_process";
+
+export interface ResolveSharedChromeProfileDirOptions {
+  envNames?: string[];
+  appDataDirName?: string;
+  profileDirName?: string;
+}
+
+export interface FindExistingChromeDebugPortOptions {
+  profileDir: string;
+  timeoutMs?: number;
+}
+
+interface ChromeVersionResponse {
+  webSocketDebuggerUrl?: string;
+}
+
+function resolveDataBaseDir(): string {
+  if (process.platform === "darwin") {
+    return path.join(os.homedir(), "Library", "Application Support");
+  }
+  if (process.platform === "win32") {
+    return process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming");
+  }
+  return process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share");
+}
+
+export function resolveSharedChromeProfileDir(
+  options: ResolveSharedChromeProfileDirOptions = {},
+): string {
+  for (const envName of options.envNames ?? []) {
+    const override = process.env[envName]?.trim();
+    if (override) {
+      return path.resolve(override);
+    }
+  }
+
+  const appDataDirName = options.appDataDirName ?? "baoyu-skills";
+  const profileDirName = options.profileDirName ?? "chrome-profile";
+  return path.join(resolveDataBaseDir(), appDataDirName, profileDirName);
+}
+
+export function resolveChromeProfileDir(profileDir?: string): string {
+  if (profileDir?.trim()) {
+    return path.resolve(profileDir.trim());
+  }
+
+  return resolveSharedChromeProfileDir({
+    envNames: ["BAOYU_CHROME_PROFILE_DIR"],
+    appDataDirName: "baoyu-skills",
+    profileDirName: "chrome-profile",
+  });
+}
+
+export function ensureChromeProfileDir(profileDir: string): string {
+  fs.mkdirSync(profileDir, { recursive: true });
+  return profileDir;
+}
+
+async function fetchWithTimeout(url: string, timeoutMs = 3_000): Promise<Response> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fetch(url, {
+      redirect: "follow",
+      signal: controller.signal,
+    });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function fetchJson<T>(url: string, timeoutMs = 3_000): Promise<T> {
+  const response = await fetchWithTimeout(url, timeoutMs);
+  if (!response.ok) {
+    throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
+  try {
+    const version = await fetchJson<ChromeVersionResponse>(`http://127.0.0.1:${port}/json/version`, timeoutMs);
+    return Boolean(version.webSocketDebuggerUrl);
+  } catch {
+    return false;
+  }
+}
+
+function parseDevToolsActivePort(filePath: string): { port: number; wsPath: string } | null {
+  try {
+    const content = fs.readFileSync(filePath, "utf8");
+    const lines = content.split(/\r?\n/);
+    const port = Number.parseInt(lines[0]?.trim() ?? "", 10);
+    const wsPath = lines[1]?.trim() ?? "";
+    if (port > 0 && wsPath) {
+      return { port, wsPath };
+    }
+  } catch {
+    // Ignore and fall back to process inspection.
+  }
+
+  return null;
+}
+
+export async function findExistingChromeDebugPort(
+  options: FindExistingChromeDebugPortOptions,
+): Promise<number | null> {
+  const timeoutMs = options.timeoutMs ?? 3_000;
+  const activePort = parseDevToolsActivePort(path.join(options.profileDir, "DevToolsActivePort"));
+  if (activePort && await isDebugPortReady(activePort.port, timeoutMs)) {
+    return activePort.port;
+  }
+
+  if (process.platform === "win32") {
+    return null;
+  }
+
+  try {
+    const result = spawnSync("ps", ["aux"], {
+      encoding: "utf8",
+      timeout: 5_000,
+    });
+    if (result.status !== 0 || !result.stdout) {
+      return null;
+    }
+
+    const lines = result.stdout
+      .split("\n")
+      .filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
+
+    for (const line of lines) {
+      const match = line.match(/--remote-debugging-port=(\d+)/);
+      const port = Number.parseInt(match?.[1] ?? "", 10);
+      if (port > 0 && await isDebugPortReady(port, timeoutMs)) {
+        return port;
+      }
+    }
+  } catch {
+    // Ignore and report no reusable debugger.
+  }
+
+  return null;
+}

packages/baoyu-fetch/src/browser/session.ts

@@ -0,0 +1,155 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { CdpClient, TargetSession, evaluateRuntime } from "./cdp-client";
+
+interface NavigationResult {
+  errorText?: string;
+}
+
+const execFileAsync = promisify(execFile);
+const MACOS_BROWSER_APP_IDS = [
+  "com.google.Chrome",
+  "org.chromium.Chromium",
+  "com.brave.Browser",
+  "com.microsoft.edgemac",
+];
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function activateBrowserApp(): Promise<void> {
+  if (process.platform !== "darwin") {
+    return;
+  }
+
+  for (const appId of MACOS_BROWSER_APP_IDS) {
+    try {
+      await execFileAsync("osascript", ["-e", `tell application id "${appId}" to activate`]);
+      return;
+    } catch {
+      // Try the next installed browser bundle id.
+    }
+  }
+}
+
+export class BrowserSession {
+  private constructor(
+    private readonly cdp: CdpClient,
+    public readonly targetSession: TargetSession,
+    public readonly interactive: boolean,
+  ) {}
+
+  static async open(
+    cdp: CdpClient,
+    options: {
+      initialUrl?: string;
+      interactive?: boolean;
+    } = {},
+  ): Promise<BrowserSession> {
+    const targetSession = await cdp.createPageSession({
+      initialUrl: options.initialUrl,
+      visible: options.interactive,
+    });
+    const browser = new BrowserSession(cdp, targetSession, Boolean(options.interactive));
+    if (browser.interactive) {
+      await browser.bringToFront().catch(() => {});
+    }
+    return browser;
+  }
+
+  async goto(url: string, timeoutMs = 30_000): Promise<void> {
+    const loadPromise = this.targetSession.waitForEvent("Page.loadEventFired", undefined, timeoutMs).catch(() => null);
+    const result = await this.targetSession.send<NavigationResult>("Page.navigate", { url });
+    if (result.errorText) {
+      throw new Error(`Navigation failed: ${result.errorText}`);
+    }
+    await loadPromise;
+    await this.waitForReadyState(timeoutMs);
+  }
+
+  async waitForReadyState(timeoutMs = 30_000): Promise<void> {
+    const startedAt = Date.now();
+    while (Date.now() - startedAt < timeoutMs) {
+      const state = await this.evaluate<string>("document.readyState");
+      if (state === "interactive" || state === "complete") {
+        return;
+      }
+      await sleep(150);
+    }
+    throw new Error("Timed out waiting for document.readyState");
+  }
+
+  async evaluate<T>(expression: string): Promise<T> {
+    return evaluateRuntime<T>(this.targetSession, expression);
+  }
+
+  async getHTML(): Promise<string> {
+    return this.evaluate<string>("document.documentElement.outerHTML");
+  }
+
+  async getTitle(): Promise<string> {
+    return this.evaluate<string>("document.title");
+  }
+
+  async getURL(): Promise<string> {
+    return this.evaluate<string>("window.location.href");
+  }
+
+  async bringToFront(): Promise<void> {
+    await this.targetSession.send("Page.bringToFront").catch(async () => {
+      await this.cdp.sendBrowserCommand("Target.activateTarget", {
+        targetId: this.targetSession.targetId,
+      });
+    });
+    if (this.interactive) {
+      await activateBrowserApp().catch(() => {});
+    }
+  }
+
+  async click(selector: string): Promise<void> {
+    const result = await this.evaluate<{ ok: boolean; error?: string }>(`
+      (() => {
+        const element = document.querySelector(${JSON.stringify(selector)});
+        if (!element) {
+          return { ok: false, error: "Element not found" };
+        }
+        element.scrollIntoView({ block: "center", inline: "center" });
+        if (element instanceof HTMLElement) {
+          element.click();
+          return { ok: true };
+        }
+        return { ok: false, error: "Element is not clickable" };
+      })()
+    `);
+
+    if (!result.ok) {
+      throw new Error(result.error ?? `Failed to click ${selector}`);
+    }
+  }
+
+  async scrollToEnd(options: { stepPx?: number; delayMs?: number; maxSteps?: number } = {}): Promise<void> {
+    const stepPx = options.stepPx ?? 1_400;
+    const delayMs = options.delayMs ?? 250;
+    const maxSteps = options.maxSteps ?? 6;
+
+    for (let step = 0; step < maxSteps; step += 1) {
+      const done = await this.evaluate<boolean>(`
+        (() => {
+          const before = window.scrollY;
+          window.scrollBy(0, ${stepPx});
+          const atBottom = window.innerHeight + window.scrollY >= document.body.scrollHeight - 4;
+          return atBottom || window.scrollY === before;
+        })()
+      `);
+      if (done) {
+        break;
+      }
+      await sleep(delayMs);
+    }
+  }
+
+  async close(): Promise<void> {
+    await this.cdp.closeTarget(this.targetSession.targetId);
+  }
+}