mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-08-02 23:29:47 +08:00
feat(baoyu-fetch): add URL reader CLI with Chrome CDP and site adapters
This commit is contained in:
Jim Liu 宝玉
@@ -0,0 +1,161 @@
|
||||
import path from "node:path";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
buildFileName,
|
||||
isDataUri,
|
||||
normalizeContentType,
|
||||
normalizeMediaUrl,
|
||||
resolveExtensionFromContentType,
|
||||
resolveExtensionFromUrl,
|
||||
resolveMediaKind,
|
||||
resolveOutputExtension,
|
||||
toPosixPath,
|
||||
} from "./media-utils";
|
||||
import type { MediaAsset, MediaDownloadRequest, MediaDownloadResult, MediaKind } from "./types";
|
||||
|
||||
const DOWNLOAD_USER_AGENT =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
|
||||
|
||||
function parseBase64DataUri(rawUrl: string): { contentType: string; bytes: Buffer } | null {
|
||||
const match = rawUrl.match(/^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/i);
|
||||
if (!match?.[1] || !match[2]) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const contentType = normalizeContentType(match[1]);
|
||||
if (!contentType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const bytes = Buffer.from(match[2].replace(/\s+/g, ""), "base64");
|
||||
if (bytes.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return { contentType, bytes };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function dedupeMedia(media: MediaAsset[]): MediaAsset[] {
|
||||
const deduped: MediaAsset[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of media) {
|
||||
const normalizedUrl = normalizeMediaUrl(item.url);
|
||||
if (!normalizedUrl || seen.has(normalizedUrl)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(normalizedUrl);
|
||||
deduped.push({
|
||||
...item,
|
||||
url: normalizedUrl,
|
||||
});
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function toRelativePath(fromDir: string, absoluteTarget: string): string {
|
||||
const relative = path.relative(fromDir, absoluteTarget) || path.basename(absoluteTarget);
|
||||
return toPosixPath(relative);
|
||||
}
|
||||
|
||||
export async function downloadMediaAssets(
|
||||
request: MediaDownloadRequest,
|
||||
): Promise<MediaDownloadResult> {
|
||||
const dedupedMedia = dedupeMedia(request.media);
|
||||
const absoluteOutputPath = path.resolve(request.outputPath);
|
||||
const markdownDir = path.dirname(absoluteOutputPath);
|
||||
const baseDir = request.mediaDir ? path.resolve(request.mediaDir) : markdownDir;
|
||||
const replacements: MediaDownloadResult["replacements"] = [];
|
||||
|
||||
let downloadedImages = 0;
|
||||
let downloadedVideos = 0;
|
||||
|
||||
for (const asset of dedupedMedia) {
|
||||
try {
|
||||
let sourceUrl = normalizeMediaUrl(asset.url);
|
||||
let contentType = "";
|
||||
let extension: string | undefined;
|
||||
let kind: MediaKind | undefined;
|
||||
let bytes: Buffer | null = null;
|
||||
|
||||
if (isDataUri(asset.url)) {
|
||||
const parsed = parseBase64DataUri(asset.url);
|
||||
if (!parsed) {
|
||||
request.log.warn(`Skipping unsupported embedded media: ${asset.url.slice(0, 32)}...`);
|
||||
continue;
|
||||
}
|
||||
|
||||
contentType = parsed.contentType;
|
||||
extension =
|
||||
resolveExtensionFromContentType(contentType) ??
|
||||
resolveExtensionFromUrl(asset.fileNameHint ?? "");
|
||||
kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
|
||||
bytes = parsed.bytes;
|
||||
} else {
|
||||
const response = await fetch(sourceUrl, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers: {
|
||||
"user-agent": DOWNLOAD_USER_AGENT,
|
||||
...(asset.headers ?? {}),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
request.log.warn(`Skipping media (${response.status}): ${asset.url}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
sourceUrl = normalizeMediaUrl(response.url || sourceUrl);
|
||||
contentType = normalizeContentType(response.headers.get("content-type"));
|
||||
extension =
|
||||
resolveExtensionFromUrl(sourceUrl) ??
|
||||
resolveExtensionFromUrl(asset.url) ??
|
||||
resolveExtensionFromUrl(asset.fileNameHint ?? "");
|
||||
kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
|
||||
bytes = Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
if (!kind || !bytes) {
|
||||
request.log.debug(`Skipping media with unresolved kind: ${asset.url}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const outputExtension = resolveOutputExtension(contentType, extension, kind);
|
||||
const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1;
|
||||
const dirName = kind === "image" ? "imgs" : "videos";
|
||||
const targetDir = path.join(baseDir, dirName);
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
|
||||
const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension, asset.fileNameHint);
|
||||
const absolutePath = path.join(targetDir, fileName);
|
||||
await writeFile(absolutePath, bytes);
|
||||
|
||||
replacements.push({
|
||||
url: asset.url,
|
||||
localPath: toRelativePath(markdownDir, absolutePath),
|
||||
absolutePath,
|
||||
kind,
|
||||
});
|
||||
|
||||
if (kind === "image") {
|
||||
downloadedImages = nextIndex;
|
||||
} else {
|
||||
downloadedVideos = nextIndex;
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
request.log.warn(`Failed to download media ${asset.url}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
replacements,
|
||||
downloadedImages,
|
||||
downloadedVideos,
|
||||
imageDir: downloadedImages > 0 ? path.join(baseDir, "imgs") : null,
|
||||
videoDir: downloadedVideos > 0 ? path.join(baseDir, "videos") : null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,458 @@
|
||||
import remarkGfm from "remark-gfm";
|
||||
import remarkParse from "remark-parse";
|
||||
import { unified } from "unified";
|
||||
import type { ContentBlock, ExtractedDocument } from "../extract/document";
|
||||
import {
|
||||
isDataUri,
|
||||
normalizeContentType,
|
||||
normalizeMediaUrl,
|
||||
resolveExtensionFromContentType,
|
||||
resolveExtensionFromUrl,
|
||||
resolveKindFromExtension,
|
||||
} from "./media-utils";
|
||||
import type { MediaAsset, MediaReplacement } from "./types";
|
||||
|
||||
const MARKDOWN_LINK_RE =
|
||||
/(!?\[[^\]\n]*\])\((<)?((?:https?:\/\/[^)\s>]+)|(?:data:[^)>\s]+))(>)?\)/g;
|
||||
const FRONTMATTER_COVER_RE = /^(coverImage:\s*")((?:https?:\/\/[^"]+)|(?:data:[^"]+))(")/m;
|
||||
const RAW_URL_RE = /(?:https?:\/\/[^\s<>"')\]]+|data:[^\s<>"')\]]+)/g;
|
||||
|
||||
interface MarkdownAstNode {
|
||||
type: string;
|
||||
url?: string | null;
|
||||
alt?: string | null;
|
||||
title?: string | null;
|
||||
value?: string | null;
|
||||
children?: MarkdownAstNode[];
|
||||
position?: {
|
||||
start?: { offset?: number | null };
|
||||
end?: { offset?: number | null };
|
||||
};
|
||||
}
|
||||
|
||||
interface MarkdownReplacementRange {
|
||||
start: number;
|
||||
end: number;
|
||||
value: string;
|
||||
}
|
||||
|
||||
function inferMediaKindFromLabel(label: string, rawUrl: string): "image" | "video" | undefined {
|
||||
if (label.startsWith("![")) {
|
||||
return "image";
|
||||
}
|
||||
|
||||
const normalizedLabel = label.replace(/[!\[\]]/g, "").trim().toLowerCase();
|
||||
if (/\b(video|animated[_ -]?gif|gif)\b/.test(normalizedLabel)) {
|
||||
return "video";
|
||||
}
|
||||
|
||||
if (isDataUri(rawUrl)) {
|
||||
const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";")));
|
||||
return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined;
|
||||
}
|
||||
|
||||
return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl));
|
||||
}
|
||||
|
||||
function inferMediaKindFromRawUrl(rawUrl: string): "image" | "video" | undefined {
|
||||
if (isDataUri(rawUrl)) {
|
||||
const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";")));
|
||||
return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined;
|
||||
}
|
||||
|
||||
return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl));
|
||||
}
|
||||
|
||||
function pushMedia(assets: MediaAsset[], seen: Set<string>, media: MediaAsset): void {
|
||||
const normalizedUrl = normalizeMediaUrl(media.url);
|
||||
if (!normalizedUrl || seen.has(normalizedUrl)) {
|
||||
return;
|
||||
}
|
||||
seen.add(normalizedUrl);
|
||||
assets.push({
|
||||
...media,
|
||||
url: normalizedUrl,
|
||||
});
|
||||
}
|
||||
|
||||
function getNodeOffsets(node: MarkdownAstNode): { start: number; end: number } | null {
|
||||
const start = node.position?.start?.offset;
|
||||
const end = node.position?.end?.offset;
|
||||
if (typeof start !== "number" || typeof end !== "number" || start < 0 || end < start) {
|
||||
return null;
|
||||
}
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
function escapeMarkdownLabel(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/\[/g, "\\[").replace(/\]/g, "\\]");
|
||||
}
|
||||
|
||||
function escapeMarkdownTitle(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
||||
}
|
||||
|
||||
function formatMarkdownDestination(url: string): string {
|
||||
return /[\s()<>]/.test(url) ? `<${url}>` : url;
|
||||
}
|
||||
|
||||
function serializeImageNode(node: MarkdownAstNode): string {
|
||||
const rawUrl = node.url ?? "";
|
||||
const normalizedUrl = normalizeMediaUrl(rawUrl);
|
||||
const alt = escapeMarkdownLabel(node.alt ?? "");
|
||||
const title = node.title ? ` "${escapeMarkdownTitle(node.title)}"` : "";
|
||||
return `}${title})`;
|
||||
}
|
||||
|
||||
function serializeLinkedImageNode(linkNode: MarkdownAstNode, imageNode: MarkdownAstNode): string {
|
||||
const imageMarkdown = serializeImageNode(imageNode);
|
||||
const imageUrl = normalizeMediaUrl(imageNode.url ?? "");
|
||||
const linkUrl = normalizeMediaUrl(linkNode.url ?? "");
|
||||
|
||||
if (!linkUrl || linkUrl === imageUrl) {
|
||||
return imageMarkdown;
|
||||
}
|
||||
|
||||
const title = linkNode.title ? ` "${escapeMarkdownTitle(linkNode.title)}"` : "";
|
||||
return `[${imageMarkdown}](${formatMarkdownDestination(linkUrl)}${title})`;
|
||||
}
|
||||
|
||||
function isParagraphWithSingleText(node: MarkdownAstNode | undefined, expectedValue: string): boolean {
|
||||
if (node?.type !== "paragraph" || node.children?.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const child = node.children[0];
|
||||
return child?.type === "text" && child.value?.trim() === expectedValue;
|
||||
}
|
||||
|
||||
function getSingleImageFromParagraph(node: MarkdownAstNode | undefined): MarkdownAstNode | null {
|
||||
if (node?.type !== "paragraph" || node.children?.length !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return node.children[0]?.type === "image" ? node.children[0] : null;
|
||||
}
|
||||
|
||||
function extractBrokenLinkedImageDestination(node: MarkdownAstNode | undefined): string | null {
|
||||
if (node?.type !== "paragraph") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const children = node.children ?? [];
|
||||
if (children.length !== 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [prefix, linkNode, suffix] = children;
|
||||
if (prefix?.type !== "text" || prefix.value?.trim() !== "](") {
|
||||
return null;
|
||||
}
|
||||
if (linkNode?.type !== "link" || !linkNode.url) {
|
||||
return null;
|
||||
}
|
||||
if (suffix?.type !== "text" || suffix.value?.trim() !== ")") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return linkNode.url;
|
||||
}
|
||||
|
||||
function collectLinkedImageReplacements(
|
||||
node: MarkdownAstNode,
|
||||
replacements: MarkdownReplacementRange[],
|
||||
): void {
|
||||
const children = node.children ?? [];
|
||||
|
||||
if (node.type === "link" && children.length === 1 && children[0]?.type === "image") {
|
||||
const offsets = getNodeOffsets(node);
|
||||
if (offsets) {
|
||||
replacements.push({
|
||||
start: offsets.start,
|
||||
end: offsets.end,
|
||||
value: serializeLinkedImageNode(node, children[0]),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (const child of children) {
|
||||
collectLinkedImageReplacements(child, replacements);
|
||||
}
|
||||
}
|
||||
|
||||
function collectBrokenLinkedImageReplacements(
|
||||
node: MarkdownAstNode,
|
||||
replacements: MarkdownReplacementRange[],
|
||||
): void {
|
||||
const children = node.children ?? [];
|
||||
for (let index = 0; index <= children.length - 3; index += 1) {
|
||||
const openParagraph = children[index];
|
||||
const imageParagraph = children[index + 1];
|
||||
const closeParagraph = children[index + 2];
|
||||
|
||||
if (!isParagraphWithSingleText(openParagraph, "[")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const imageNode = getSingleImageFromParagraph(imageParagraph);
|
||||
if (!imageNode) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const linkUrl = extractBrokenLinkedImageDestination(closeParagraph);
|
||||
if (!linkUrl) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const start = openParagraph.position?.start?.offset;
|
||||
const end = closeParagraph.position?.end?.offset;
|
||||
if (typeof start !== "number" || typeof end !== "number" || end < start) {
|
||||
continue;
|
||||
}
|
||||
|
||||
replacements.push({
|
||||
start,
|
||||
end,
|
||||
value: serializeLinkedImageNode({ type: "link", url: linkUrl }, imageNode),
|
||||
});
|
||||
|
||||
index += 2;
|
||||
}
|
||||
|
||||
for (const child of children) {
|
||||
collectBrokenLinkedImageReplacements(child, replacements);
|
||||
}
|
||||
}
|
||||
|
||||
function applyReplacements(source: string, replacements: MarkdownReplacementRange[]): string {
|
||||
if (replacements.length === 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
let result = source;
|
||||
const sorted = [...replacements].sort((left, right) => right.start - left.start);
|
||||
for (const replacement of sorted) {
|
||||
result = `${result.slice(0, replacement.start)}${replacement.value}${result.slice(replacement.end)}`;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeLinkedImageMarkdown(markdown: string): string {
|
||||
let tree: MarkdownAstNode;
|
||||
try {
|
||||
tree = unified().use(remarkParse).use(remarkGfm).parse(markdown) as MarkdownAstNode;
|
||||
} catch {
|
||||
return markdown;
|
||||
}
|
||||
|
||||
const replacements: MarkdownReplacementRange[] = [];
|
||||
collectLinkedImageReplacements(tree, replacements);
|
||||
collectBrokenLinkedImageReplacements(tree, replacements);
|
||||
return applyReplacements(markdown, replacements);
|
||||
}
|
||||
|
||||
export function normalizeMarkdownMediaLinks(markdown: string): string {
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, openAngle, rawUrl, closeAngle) => {
|
||||
const normalizedUrl = normalizeMediaUrl(rawUrl);
|
||||
if (normalizedUrl === rawUrl) {
|
||||
return full;
|
||||
}
|
||||
return `${label}(${openAngle ?? ""}${normalizedUrl}${closeAngle ?? ""})`;
|
||||
});
|
||||
|
||||
result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
|
||||
const normalizedUrl = normalizeMediaUrl(rawUrl);
|
||||
if (normalizedUrl === rawUrl) {
|
||||
return full;
|
||||
}
|
||||
return `${prefix}${normalizedUrl}${suffix}`;
|
||||
});
|
||||
|
||||
RAW_URL_RE.lastIndex = 0;
|
||||
result = result.replace(RAW_URL_RE, (rawUrl) => normalizeMediaUrl(rawUrl));
|
||||
return normalizeLinkedImageMarkdown(result);
|
||||
}
|
||||
|
||||
export function collectMediaFromText(
|
||||
text: string,
|
||||
options: {
|
||||
role?: MediaAsset["role"];
|
||||
defaultKind?: MediaAsset["kind"];
|
||||
seen?: Set<string>;
|
||||
into?: MediaAsset[];
|
||||
} = {},
|
||||
): MediaAsset[] {
|
||||
const assets = options.into ?? [];
|
||||
const seen = options.seen ?? new Set<string>();
|
||||
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let linkMatch: RegExpExecArray | null;
|
||||
while ((linkMatch = MARKDOWN_LINK_RE.exec(text))) {
|
||||
const label = linkMatch[1] ?? "";
|
||||
const rawUrl = linkMatch[3] ?? "";
|
||||
const kind = inferMediaKindFromLabel(label, rawUrl) ?? options.defaultKind;
|
||||
if (!kind) {
|
||||
continue;
|
||||
}
|
||||
pushMedia(assets, seen, {
|
||||
url: rawUrl,
|
||||
kind,
|
||||
role: options.role ?? "inline",
|
||||
});
|
||||
}
|
||||
|
||||
RAW_URL_RE.lastIndex = 0;
|
||||
let rawMatch: RegExpExecArray | null;
|
||||
while ((rawMatch = RAW_URL_RE.exec(text))) {
|
||||
const rawUrl = rawMatch[0] ?? "";
|
||||
const kind = inferMediaKindFromRawUrl(rawUrl) ?? options.defaultKind;
|
||||
if (!kind) {
|
||||
continue;
|
||||
}
|
||||
pushMedia(assets, seen, {
|
||||
url: rawUrl,
|
||||
kind,
|
||||
role: options.role ?? "inline",
|
||||
});
|
||||
}
|
||||
|
||||
return assets;
|
||||
}
|
||||
|
||||
function collectMediaFromBlock(
|
||||
block: ContentBlock,
|
||||
assets: MediaAsset[],
|
||||
seen: Set<string>,
|
||||
): void {
|
||||
switch (block.type) {
|
||||
case "image":
|
||||
pushMedia(assets, seen, {
|
||||
url: block.url,
|
||||
kind: "image",
|
||||
role: "inline",
|
||||
alt: block.alt,
|
||||
});
|
||||
return;
|
||||
case "html":
|
||||
case "markdown":
|
||||
collectMediaFromText(block.type === "html" ? block.html : block.markdown, {
|
||||
role: "inline",
|
||||
seen,
|
||||
into: assets,
|
||||
});
|
||||
return;
|
||||
case "paragraph":
|
||||
case "quote":
|
||||
collectMediaFromText(block.text, {
|
||||
role: "inline",
|
||||
seen,
|
||||
into: assets,
|
||||
});
|
||||
return;
|
||||
case "list":
|
||||
for (const item of block.items) {
|
||||
collectMediaFromText(item, {
|
||||
role: "attachment",
|
||||
seen,
|
||||
into: assets,
|
||||
});
|
||||
}
|
||||
return;
|
||||
case "heading":
|
||||
case "code":
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export function collectMediaFromDocument(document: ExtractedDocument): MediaAsset[] {
|
||||
const assets: MediaAsset[] = [];
|
||||
const seen = new Set<string>();
|
||||
const coverImage =
|
||||
typeof document.metadata?.coverImage === "string" ? document.metadata.coverImage : undefined;
|
||||
|
||||
if (coverImage) {
|
||||
pushMedia(assets, seen, {
|
||||
url: coverImage,
|
||||
kind: "image",
|
||||
role: "cover",
|
||||
});
|
||||
}
|
||||
|
||||
for (const block of document.content) {
|
||||
collectMediaFromBlock(block, assets, seen);
|
||||
}
|
||||
|
||||
return assets;
|
||||
}
|
||||
|
||||
export function collectMediaFromMarkdown(markdown: string): MediaAsset[] {
|
||||
const assets: MediaAsset[] = [];
|
||||
const seen = new Set<string>();
|
||||
const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (fmMatch) {
|
||||
const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
|
||||
if (coverMatch?.[2]) {
|
||||
pushMedia(assets, seen, {
|
||||
url: coverMatch[2],
|
||||
kind: "image",
|
||||
role: "cover",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
collectMediaFromText(markdown, { seen, into: assets });
|
||||
return assets;
|
||||
}
|
||||
|
||||
export function rewriteMarkdownMediaLinks(
|
||||
markdown: string,
|
||||
replacements: MediaReplacement[],
|
||||
): string {
|
||||
if (replacements.length === 0) {
|
||||
return markdown;
|
||||
}
|
||||
|
||||
const replacementMap = new Map<string, string>();
|
||||
for (const item of replacements) {
|
||||
replacementMap.set(item.url, item.localPath);
|
||||
replacementMap.set(normalizeMediaUrl(item.url), item.localPath);
|
||||
}
|
||||
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, _openAngle, rawUrl) => {
|
||||
const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl));
|
||||
if (!replacement) {
|
||||
return full;
|
||||
}
|
||||
return `${label}(${replacement})`;
|
||||
});
|
||||
|
||||
result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
|
||||
const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl));
|
||||
if (!replacement) {
|
||||
return full;
|
||||
}
|
||||
return `${prefix}${replacement}${suffix}`;
|
||||
});
|
||||
|
||||
for (const { url, localPath } of replacements) {
|
||||
result = result.split(url).join(localPath);
|
||||
const normalizedUrl = normalizeMediaUrl(url);
|
||||
if (normalizedUrl !== url) {
|
||||
result = result.split(normalizedUrl).join(localPath);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export function resolveDataUriExtension(rawUrl: string): string | undefined {
|
||||
if (!isDataUri(rawUrl)) {
|
||||
return undefined;
|
||||
}
|
||||
const separatorIndex = rawUrl.indexOf(";");
|
||||
const contentType = normalizeContentType(rawUrl.slice(5, separatorIndex === -1 ? undefined : separatorIndex));
|
||||
return resolveExtensionFromContentType(contentType);
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
import path from "node:path";
|
||||
import type { MediaKind } from "./types";
|
||||
|
||||
const IMAGE_EXTENSIONS = new Set([
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"png",
|
||||
"webp",
|
||||
"gif",
|
||||
"bmp",
|
||||
"avif",
|
||||
"heic",
|
||||
"heif",
|
||||
"svg",
|
||||
]);
|
||||
|
||||
const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]);
|
||||
|
||||
const MIME_EXTENSION_MAP: Record<string, string> = {
|
||||
"image/jpeg": "jpg",
|
||||
"image/jpg": "jpg",
|
||||
"image/png": "png",
|
||||
"image/webp": "webp",
|
||||
"image/gif": "gif",
|
||||
"image/bmp": "bmp",
|
||||
"image/avif": "avif",
|
||||
"image/heic": "heic",
|
||||
"image/heif": "heif",
|
||||
"image/svg+xml": "svg",
|
||||
"video/mp4": "mp4",
|
||||
"video/webm": "webm",
|
||||
"video/quicktime": "mov",
|
||||
"video/x-m4v": "m4v",
|
||||
};
|
||||
|
||||
export function normalizeContentType(raw: string | null): string {
|
||||
return raw?.split(";")[0]?.trim().toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
export function normalizeExtension(raw: string | undefined | null): string | undefined {
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = raw.replace(/^\./, "").trim().toLowerCase();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
if (trimmed === "jpeg" || trimmed === "jpg") {
|
||||
return "jpg";
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function resolveExtensionFromUrl(rawUrl: string): string | undefined {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname));
|
||||
if (extFromPath) {
|
||||
return extFromPath;
|
||||
}
|
||||
const extFromFormat = normalizeExtension(parsed.searchParams.get("format"));
|
||||
if (extFromFormat) {
|
||||
return extFromFormat;
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveExtensionFromContentType(contentType: string): string | undefined {
|
||||
return normalizeExtension(MIME_EXTENSION_MAP[contentType]);
|
||||
}
|
||||
|
||||
export function resolveKindFromContentType(contentType: string): MediaKind | undefined {
|
||||
if (!contentType) {
|
||||
return undefined;
|
||||
}
|
||||
if (contentType.startsWith("image/")) {
|
||||
return "image";
|
||||
}
|
||||
if (contentType.startsWith("video/")) {
|
||||
return "video";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveKindFromExtension(extension: string | undefined): MediaKind | undefined {
|
||||
if (!extension) {
|
||||
return undefined;
|
||||
}
|
||||
if (IMAGE_EXTENSIONS.has(extension)) {
|
||||
return "image";
|
||||
}
|
||||
if (VIDEO_EXTENSIONS.has(extension)) {
|
||||
return "video";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveMediaKind(
|
||||
rawUrl: string,
|
||||
contentType: string,
|
||||
extension: string | undefined,
|
||||
hint?: MediaKind,
|
||||
): MediaKind | undefined {
|
||||
const kindFromType = resolveKindFromContentType(contentType);
|
||||
if (kindFromType) {
|
||||
return kindFromType;
|
||||
}
|
||||
|
||||
const kindFromExtension = resolveKindFromExtension(extension);
|
||||
if (kindFromExtension) {
|
||||
return kindFromExtension;
|
||||
}
|
||||
|
||||
if (contentType && contentType !== "application/octet-stream") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (hint) {
|
||||
return hint;
|
||||
}
|
||||
|
||||
if (rawUrl.startsWith("data:image/")) {
|
||||
return "image";
|
||||
}
|
||||
|
||||
if (rawUrl.startsWith("data:video/")) {
|
||||
return "video";
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveOutputExtension(
|
||||
contentType: string,
|
||||
extension: string | undefined,
|
||||
kind: MediaKind,
|
||||
): string {
|
||||
const fromMime = resolveExtensionFromContentType(contentType);
|
||||
if (fromMime) {
|
||||
return fromMime;
|
||||
}
|
||||
const normalized = normalizeExtension(extension);
|
||||
if (normalized) {
|
||||
return normalized;
|
||||
}
|
||||
return kind === "video" ? "mp4" : "jpg";
|
||||
}
|
||||
|
||||
export function isDataUri(value: string): boolean {
|
||||
return value.startsWith("data:");
|
||||
}
|
||||
|
||||
export function safeDecodeURIComponent(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function extractEmbeddedUrl(value: string): string | undefined {
|
||||
const encodedMatch = value.match(/https?%3A%2F%2F.+$/i)?.[0];
|
||||
if (encodedMatch) {
|
||||
const decoded = safeDecodeURIComponent(encodedMatch);
|
||||
try {
|
||||
return new URL(decoded).href;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const literalMatch = value.match(/https?:\/\/.+$/i)?.[0];
|
||||
if (!literalMatch) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
return new URL(literalMatch).href;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeMediaUrl(rawUrl: string): string {
|
||||
if (isDataUri(rawUrl)) {
|
||||
return rawUrl;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
const hostname = parsed.hostname.toLowerCase();
|
||||
|
||||
if (hostname === "substackcdn.com" || hostname.endsWith(".substackcdn.com")) {
|
||||
const embeddedUrl = extractEmbeddedUrl(`${parsed.pathname}${parsed.search}`);
|
||||
if (embeddedUrl) {
|
||||
return embeddedUrl;
|
||||
}
|
||||
}
|
||||
|
||||
return parsed.href;
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export function sanitizeFileSegment(input: string): string {
|
||||
return input
|
||||
.replace(/[^a-zA-Z0-9_-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^[-_]+|[-_]+$/g, "")
|
||||
.slice(0, 48);
|
||||
}
|
||||
|
||||
export function resolveFileStem(rawUrl: string, extension: string, fileNameHint?: string): string {
|
||||
const hintBase = fileNameHint?.trim();
|
||||
if (hintBase) {
|
||||
const parsed = path.posix.parse(hintBase);
|
||||
const stem = parsed.name || parsed.base;
|
||||
return sanitizeFileSegment(stem);
|
||||
}
|
||||
|
||||
if (isDataUri(rawUrl)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
const base = path.posix.basename(parsed.pathname);
|
||||
if (!base) {
|
||||
return "";
|
||||
}
|
||||
const decodedBase = safeDecodeURIComponent(base);
|
||||
const normalizedExtension = normalizeExtension(extension);
|
||||
const stripExtension = normalizedExtension ? new RegExp(`\\.${normalizedExtension}$`, "i") : null;
|
||||
const rawStem = stripExtension ? decodedBase.replace(stripExtension, "") : decodedBase;
|
||||
return sanitizeFileSegment(rawStem);
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFileName(
|
||||
kind: MediaKind,
|
||||
index: number,
|
||||
sourceUrl: string,
|
||||
extension: string,
|
||||
fileNameHint?: string,
|
||||
): string {
|
||||
const stem = resolveFileStem(sourceUrl, extension, fileNameHint);
|
||||
const prefix = kind === "image" ? "img" : "video";
|
||||
const serial = String(index).padStart(3, "0");
|
||||
const suffix = stem ? `-${stem}` : "";
|
||||
return `${prefix}-${serial}${suffix}.${extension}`;
|
||||
}
|
||||
|
||||
export function toPosixPath(value: string): string {
|
||||
return value.split(path.sep).join(path.posix.sep);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import type { Logger } from "../utils/logger";
|
||||
|
||||
export type MediaKind = "image" | "video";
|
||||
|
||||
export interface MediaAsset {
|
||||
url: string;
|
||||
kind?: MediaKind;
|
||||
role?: "cover" | "inline" | "attachment";
|
||||
alt?: string;
|
||||
fileNameHint?: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface MediaReplacement {
|
||||
url: string;
|
||||
localPath: string;
|
||||
absolutePath: string;
|
||||
kind: MediaKind;
|
||||
}
|
||||
|
||||
export interface MediaDownloadRequest {
|
||||
media: MediaAsset[];
|
||||
outputPath: string;
|
||||
mediaDir?: string;
|
||||
log: Logger;
|
||||
}
|
||||
|
||||
export interface MediaDownloadResult {
|
||||
replacements: MediaReplacement[];
|
||||
downloadedImages: number;
|
||||
downloadedVideos: number;
|
||||
imageDir: string | null;
|
||||
videoDir: string | null;
|
||||
}
|
||||
Reference in New Issue
Block a user