mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-30 13:49:48 +08:00
refactor(baoyu-translate): replace remark/unified with markdown-it for chunk parsing
Simplifies dependencies and adds main.ts CLI entry point with exported functions for programmatic reuse.
This commit is contained in:
Jim Liu 宝玉
@@ -1,137 +1,335 @@
|
||||
import { readFileSync, writeFileSync, mkdirSync } from "fs"
|
||||
import { basename, dirname, join } from "path"
|
||||
import { unified } from "unified"
|
||||
import remarkParse from "remark-parse"
|
||||
import remarkGfm from "remark-gfm"
|
||||
import remarkFrontmatter from "remark-frontmatter"
|
||||
import remarkStringify from "remark-stringify"
|
||||
import type { Root, Content } from "mdast"
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "fs"
|
||||
import { dirname, join } from "path"
|
||||
import MarkdownIt from "markdown-it"
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const file = args.find(a => !a.startsWith("--"))
|
||||
const maxWords = parseInt(args[args.indexOf("--max-words") + 1] || "5000")
|
||||
const outputDir = args.indexOf("--output-dir") !== -1 ? args[args.indexOf("--output-dir") + 1] : ""
|
||||
type BlockKind =
|
||||
| "heading"
|
||||
| "thematicBreak"
|
||||
| "html"
|
||||
| "code"
|
||||
| "flow"
|
||||
|
||||
if (!file) {
|
||||
console.error("Usage: chunk.ts <file> [--max-words 5000]")
|
||||
process.exit(1)
|
||||
interface Block {
|
||||
kind: BlockKind
|
||||
md: string
|
||||
words: number
|
||||
}
|
||||
|
||||
const content = readFileSync(file, "utf-8")
|
||||
interface Chunk {
|
||||
blocks: Block[]
|
||||
words: number
|
||||
}
|
||||
|
||||
const tree = unified()
|
||||
.use(remarkParse)
|
||||
.use(remarkGfm)
|
||||
.use(remarkFrontmatter, ["yaml"])
|
||||
.parse(content)
|
||||
export interface ChunkCliOptions {
|
||||
file: string
|
||||
maxWords: number
|
||||
outputDir: string
|
||||
}
|
||||
|
||||
const stringify = unified()
|
||||
.use(remarkStringify, { bullet: "-", emphasis: "*", strong: "*" })
|
||||
.use(remarkGfm)
|
||||
.use(remarkFrontmatter, ["yaml"])
|
||||
export interface ChunkResult {
|
||||
source: string
|
||||
chunks: number
|
||||
output_dir: string
|
||||
frontmatter: boolean
|
||||
words_per_chunk: number[]
|
||||
}
|
||||
|
||||
function nodeToMd(node: Content): string {
|
||||
const root: Root = { type: "root", children: [node] }
|
||||
return stringify.stringify(root).trim()
|
||||
const parser = new MarkdownIt({ html: true })
|
||||
|
||||
export function formatChunkUsage(command: string): string {
|
||||
return `Usage: ${command} <file> [--max-words 5000] [--output-dir <dir>]`
|
||||
}
|
||||
|
||||
export function runChunkCli(args: string[], command = "chunk.ts"): number {
|
||||
const parsed = parseChunkCliArgs(args)
|
||||
|
||||
if ("help" in parsed) {
|
||||
console.log(formatChunkUsage(command))
|
||||
return 0
|
||||
}
|
||||
|
||||
if ("error" in parsed) {
|
||||
console.error(parsed.error)
|
||||
console.error(formatChunkUsage(command))
|
||||
return 1
|
||||
}
|
||||
|
||||
const result = chunkMarkdownFile(parsed.file, {
|
||||
maxWords: parsed.maxWords,
|
||||
outputDir: parsed.outputDir,
|
||||
})
|
||||
|
||||
console.log(JSON.stringify(result))
|
||||
return 0
|
||||
}
|
||||
|
||||
export function chunkMarkdownFile(
|
||||
file: string,
|
||||
options: { maxWords?: number; outputDir?: string } = {}
|
||||
): ChunkResult {
|
||||
const maxWords = options.maxWords ?? 5000
|
||||
const outputDir = options.outputDir ?? ""
|
||||
|
||||
const rawContent = normalizeNewlines(readFileSync(file, "utf-8"))
|
||||
const { frontmatter, body } = extractFrontmatter(rawContent)
|
||||
const chunks = buildChunks(parseMarkdown(body), maxWords)
|
||||
|
||||
const dir = outputDir ? join(outputDir, "chunks") : join(dirname(file), "chunks")
|
||||
mkdirSync(dir, { recursive: true })
|
||||
|
||||
if (frontmatter) {
|
||||
writeFileSync(join(dir, "frontmatter.md"), frontmatter)
|
||||
}
|
||||
|
||||
chunks.forEach((chunk, index) => {
|
||||
const num = String(index + 1).padStart(2, "0")
|
||||
writeFileSync(join(dir, `chunk-${num}.md`), chunk.blocks.map(block => block.md).join("\n\n"))
|
||||
})
|
||||
|
||||
return {
|
||||
source: file,
|
||||
chunks: chunks.length,
|
||||
output_dir: dir,
|
||||
frontmatter: Boolean(frontmatter),
|
||||
words_per_chunk: chunks.map(chunk => chunk.words),
|
||||
}
|
||||
}
|
||||
|
||||
function parseChunkCliArgs(args: string[]):
|
||||
| ChunkCliOptions
|
||||
| { help: true }
|
||||
| { error: string } {
|
||||
let file = ""
|
||||
let maxWords = 5000
|
||||
let outputDir = ""
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index]
|
||||
|
||||
if (arg === "-h" || arg === "--help") {
|
||||
return { help: true }
|
||||
}
|
||||
|
||||
if (arg === "--max-words") {
|
||||
const value = args[index + 1]
|
||||
if (!value) return { error: "Missing value for --max-words" }
|
||||
maxWords = parsePositiveInt(value, 0)
|
||||
if (maxWords <= 0) return { error: `Invalid --max-words value: ${value}` }
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg === "--output-dir") {
|
||||
const value = args[index + 1]
|
||||
if (!value) return { error: "Missing value for --output-dir" }
|
||||
outputDir = value
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg.startsWith("-")) {
|
||||
return { error: `Unknown option: ${arg}` }
|
||||
}
|
||||
|
||||
if (!file) {
|
||||
file = arg
|
||||
continue
|
||||
}
|
||||
|
||||
return { error: `Unexpected positional argument: ${arg}` }
|
||||
}
|
||||
|
||||
if (!file) {
|
||||
return { error: "Missing input file" }
|
||||
}
|
||||
|
||||
return { file, maxWords, outputDir }
|
||||
}
|
||||
|
||||
function parsePositiveInt(value: string | undefined, fallback: number): number {
|
||||
if (!value) return fallback
|
||||
const parsed = Number.parseInt(value, 10)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
|
||||
}
|
||||
|
||||
function normalizeNewlines(text: string): string {
|
||||
return text.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n")
|
||||
}
|
||||
|
||||
function trimBoundaryBlankLines(text: string): string {
|
||||
return text.replace(/^\n+/, "").replace(/\n+$/, "")
|
||||
}
|
||||
|
||||
function extractFrontmatter(content: string): { frontmatter: string; body: string } {
|
||||
const lines = content.split("\n")
|
||||
if (lines[0] !== "---") {
|
||||
return { frontmatter: "", body: content }
|
||||
}
|
||||
|
||||
for (let index = 1; index < lines.length; index += 1) {
|
||||
if (lines[index] === "---" || lines[index] === "...") {
|
||||
return {
|
||||
frontmatter: lines.slice(0, index + 1).join("\n"),
|
||||
body: lines.slice(index + 1).join("\n").replace(/^\n+/, ""),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { frontmatter: "", body: content }
|
||||
}
|
||||
|
||||
function parseMarkdown(content: string): Block[] {
|
||||
if (!content.trim()) return []
|
||||
|
||||
const lines = content.split("\n")
|
||||
const tokens = parser.parse(content, {})
|
||||
const blocks: Block[] = []
|
||||
|
||||
for (const token of tokens) {
|
||||
if (!token.map || token.level !== 0) continue
|
||||
if (token.nesting !== 1 && token.nesting !== 0) continue
|
||||
|
||||
const [startLine, endLine] = token.map
|
||||
const md = trimBoundaryBlankLines(lines.slice(startLine, endLine).join("\n"))
|
||||
if (!md) continue
|
||||
|
||||
blocks.push(makeBlock(tokenTypeToBlockKind(token.type), md))
|
||||
}
|
||||
|
||||
if (blocks.length === 0) {
|
||||
const body = trimBoundaryBlankLines(content)
|
||||
if (body) {
|
||||
blocks.push(makeBlock("flow", body))
|
||||
}
|
||||
}
|
||||
|
||||
return blocks
|
||||
}
|
||||
|
||||
function tokenTypeToBlockKind(tokenType: string): BlockKind {
|
||||
if (tokenType === "heading_open") return "heading"
|
||||
if (tokenType === "hr") return "thematicBreak"
|
||||
if (tokenType === "html_block") return "html"
|
||||
if (tokenType === "fence" || tokenType === "code_block") return "code"
|
||||
return "flow"
|
||||
}
|
||||
|
||||
function makeBlock(kind: BlockKind, md: string): Block {
|
||||
return {
|
||||
kind,
|
||||
md: trimBoundaryBlankLines(md),
|
||||
words: countWords(md),
|
||||
}
|
||||
}
|
||||
|
||||
function buildChunks(blocks: Block[], maxWordsPerChunk: number): Chunk[] {
|
||||
const sections = splitIntoSections(blocks)
|
||||
const normalizedBlocks: Block[] = []
|
||||
|
||||
for (const section of sections) {
|
||||
const sectionWords = section.reduce((sum, block) => sum + block.words, 0)
|
||||
if (sectionWords <= maxWordsPerChunk) {
|
||||
normalizedBlocks.push(makeBlock("flow", section.map(block => block.md).join("\n\n")))
|
||||
continue
|
||||
}
|
||||
|
||||
for (const block of section) {
|
||||
normalizedBlocks.push(...splitOversizedBlock(block, maxWordsPerChunk))
|
||||
}
|
||||
}
|
||||
|
||||
const chunks: Chunk[] = []
|
||||
let currentBlocks: Block[] = []
|
||||
let currentWords = 0
|
||||
|
||||
for (const block of normalizedBlocks) {
|
||||
if (currentWords + block.words > maxWordsPerChunk && currentBlocks.length > 0) {
|
||||
chunks.push({ blocks: currentBlocks, words: currentWords })
|
||||
currentBlocks = [block]
|
||||
currentWords = block.words
|
||||
continue
|
||||
}
|
||||
|
||||
currentBlocks.push(block)
|
||||
currentWords += block.words
|
||||
}
|
||||
|
||||
if (currentBlocks.length > 0) {
|
||||
chunks.push({ blocks: currentBlocks, words: currentWords })
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
function splitIntoSections(blocks: Block[]): Block[][] {
|
||||
const sections: Block[][] = []
|
||||
let current: Block[] = []
|
||||
|
||||
for (const block of blocks) {
|
||||
if (block.kind === "heading" && current.length > 0) {
|
||||
sections.push(current)
|
||||
current = [block]
|
||||
continue
|
||||
}
|
||||
|
||||
current.push(block)
|
||||
}
|
||||
|
||||
if (current.length > 0) {
|
||||
sections.push(current)
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
function splitOversizedBlock(block: Block, maxWordsPerChunk: number): Block[] {
|
||||
if (block.words <= maxWordsPerChunk) return [block]
|
||||
|
||||
if (
|
||||
block.kind === "heading"
|
||||
|| block.kind === "thematicBreak"
|
||||
|| block.kind === "html"
|
||||
|| block.kind === "code"
|
||||
) {
|
||||
return [block]
|
||||
}
|
||||
|
||||
const lines = block.md.split("\n")
|
||||
if (lines.length <= 1) {
|
||||
return [block]
|
||||
}
|
||||
|
||||
const splitBlocks: Block[] = []
|
||||
let buffer: string[] = []
|
||||
let bufferWords = 0
|
||||
|
||||
for (const line of lines) {
|
||||
const lineWords = countWords(line)
|
||||
if (bufferWords + lineWords > maxWordsPerChunk && buffer.length > 0) {
|
||||
splitBlocks.push(makeBlock(block.kind, buffer.join("\n")))
|
||||
buffer = [line]
|
||||
bufferWords = lineWords
|
||||
continue
|
||||
}
|
||||
|
||||
buffer.push(line)
|
||||
bufferWords += lineWords
|
||||
}
|
||||
|
||||
if (buffer.length > 0) {
|
||||
splitBlocks.push(makeBlock(block.kind, buffer.join("\n")))
|
||||
}
|
||||
|
||||
return splitBlocks
|
||||
}
|
||||
|
||||
function countWords(text: string): number {
|
||||
const cleaned = text.replace(/[#*`\[\]()>|_~-]/g, " ")
|
||||
const cjk = cleaned.match(/[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/g)
|
||||
const latin = cleaned.match(/[a-zA-Z0-9]+/g)
|
||||
return (cjk?.length || 0) + (latin?.length || 0)
|
||||
return (cjk?.length ?? 0) + (latin?.length ?? 0)
|
||||
}
|
||||
|
||||
interface Block {
|
||||
md: string
|
||||
words: number
|
||||
if (import.meta.main) {
|
||||
process.exit(runChunkCli(process.argv.slice(2), process.argv[1] ?? "chunk.ts"))
|
||||
}
|
||||
|
||||
function splitNodeToBlocks(node: Content): Block[] {
|
||||
const md = nodeToMd(node)
|
||||
const words = countWords(md)
|
||||
|
||||
if (words <= maxWords) return [{ md, words }]
|
||||
|
||||
if (node.type === "heading" || node.type === "thematicBreak" || node.type === "html") {
|
||||
return [{ md, words }]
|
||||
}
|
||||
|
||||
if ("children" in node && Array.isArray(node.children)) {
|
||||
const blocks: Block[] = []
|
||||
for (const child of node.children as Content[]) {
|
||||
blocks.push(...splitNodeToBlocks(child))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
const lines = md.split("\n")
|
||||
if (lines.length > 1) {
|
||||
const blocks: Block[] = []
|
||||
let buf: string[] = []
|
||||
let bufWords = 0
|
||||
for (const line of lines) {
|
||||
const lw = countWords(line)
|
||||
if (bufWords + lw > maxWords && buf.length > 0) {
|
||||
blocks.push({ md: buf.join("\n"), words: bufWords })
|
||||
buf = [line]
|
||||
bufWords = lw
|
||||
} else {
|
||||
buf.push(line)
|
||||
bufWords += lw
|
||||
}
|
||||
}
|
||||
if (buf.length > 0) blocks.push({ md: buf.join("\n"), words: bufWords })
|
||||
return blocks
|
||||
}
|
||||
|
||||
return [{ md, words }]
|
||||
}
|
||||
|
||||
let frontmatter = ""
|
||||
const blocks: Block[] = []
|
||||
|
||||
for (const node of tree.children) {
|
||||
if (node.type === "yaml") {
|
||||
frontmatter = `---\n${node.value}\n---`
|
||||
continue
|
||||
}
|
||||
blocks.push(...splitNodeToBlocks(node as Content))
|
||||
}
|
||||
|
||||
const chunks: { blocks: Block[]; words: number }[] = []
|
||||
let cur: Block[] = []
|
||||
let curWords = 0
|
||||
|
||||
for (const b of blocks) {
|
||||
if (curWords + b.words > maxWords && cur.length > 0) {
|
||||
chunks.push({ blocks: cur, words: curWords })
|
||||
cur = [b]
|
||||
curWords = b.words
|
||||
} else {
|
||||
cur.push(b)
|
||||
curWords += b.words
|
||||
}
|
||||
}
|
||||
if (cur.length > 0) chunks.push({ blocks: cur, words: curWords })
|
||||
|
||||
const dir = outputDir ? join(outputDir, "chunks") : join(dirname(file), "chunks")
|
||||
mkdirSync(dir, { recursive: true })
|
||||
|
||||
if (frontmatter) {
|
||||
writeFileSync(join(dir, "frontmatter.md"), frontmatter)
|
||||
}
|
||||
|
||||
chunks.forEach((chunk, i) => {
|
||||
const num = String(i + 1).padStart(2, "0")
|
||||
const out = join(dir, `chunk-${num}.md`)
|
||||
writeFileSync(out, chunk.blocks.map(b => b.md).join("\n\n"))
|
||||
})
|
||||
|
||||
console.log(JSON.stringify({
|
||||
source: file,
|
||||
chunks: chunks.length,
|
||||
output_dir: dir,
|
||||
frontmatter: !!frontmatter,
|
||||
words_per_chunk: chunks.map(c => c.words)
|
||||
}))
|
||||
|
||||
Reference in New Issue
Block a user