diff --git a/docs/publishing.md b/docs/publishing.md index 10849c7..1008d7c 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -21,18 +21,18 @@ bash scripts/sync-clawhub.sh # sync all skills bash scripts/sync-clawhub.sh # sync one skill ``` -Release hooks are configured via `.releaserc.yml`. This repo does not stage a separate release directory: release prep verifies that skills depend on published npm package versions, and publish reads the skill directory directly. +Release hooks are configured via `.releaserc.yml`. This repo does not stage a separate release directory: publish reads the skill directory directly and validates that local package references and CLI bin targets are self-contained. ## Shared Workspace Packages -`packages/` is the **only** source of truth for shared runtime code. Publish shared packages to npm and reference them from skill script `package.json` files with semver ranges. Do not vendor shared packages into `skills/*/scripts/vendor/`. +`packages/` is the source of truth for shared runtime code. Most skills consume shared packages from npm with semver ranges. `baoyu-url-to-markdown` is the exception: it vendors the `baoyu-fetch` runtime into `skills/baoyu-url-to-markdown/scripts/lib/` so the published skill is self-contained and does not depend on the `baoyu-fetch` npm package. Current packages: - `baoyu-chrome-cdp` (Chrome CDP utilities), consumed by 5 skills (`baoyu-danger-gemini-web`, `baoyu-danger-x-to-markdown`, `baoyu-post-to-wechat`, `baoyu-post-to-weibo`, `baoyu-post-to-x`) - `baoyu-md` (shared Markdown rendering and placeholder pipeline), consumed by 3 skills (`baoyu-markdown-to-html`, `baoyu-post-to-wechat`, `baoyu-post-to-weibo`) -- `baoyu-fetch` (URL-to-Markdown CLI), consumed by 1 skill (`baoyu-url-to-markdown`) +- `baoyu-fetch` (URL-to-Markdown CLI), vendored into 1 skill (`baoyu-url-to-markdown`) -**How it works**: npm packages are built from `packages/` and published to the public npm registry. Skills depend on those packages with `^` specs. Release prep runs `node scripts/verify-shared-package-deps.mjs` so `file:` dependencies and vendored workspace packages cannot slip back in. +**How it works**: npm packages are built from `packages/` and published to the public npm registry. Skills normally depend on those packages with `^` specs. Release prep runs `node scripts/verify-shared-package-deps.mjs` so accidental `file:` dependencies cannot slip back in. For vendored skill runtimes, keep the copied code under the skill directory and run `node scripts/publish-skill.mjs --skill-dir --version --dry-run` before publishing. **Update workflow**: 1. Edit package under `packages/` diff --git a/scripts/lib/release-files.mjs b/scripts/lib/release-files.mjs index 53eb230..b05640c 100644 --- a/scripts/lib/release-files.mjs +++ b/scripts/lib/release-files.mjs @@ -10,6 +10,20 @@ const PACKAGE_DEPENDENCY_SECTIONS = [ const SKIPPED_DIRS = new Set([".git", ".clawhub", ".clawdhub", "node_modules", "out", "dist", "build"]); const SKIPPED_FILES = new Set([".DS_Store", "bun.lockb"]); +const MIME_MAP = { + ".md": "text/markdown", + ".ts": "text/plain", + ".js": "text/javascript", + ".mjs": "text/javascript", + ".json": "application/json", + ".yml": "text/yaml", + ".yaml": "text/yaml", + ".txt": "text/plain", + ".html": "text/html", + ".css": "text/css", + ".xml": "text/xml", + ".svg": "image/svg+xml", +}; export async function listReleaseFiles(root) { const resolvedRoot = path.resolve(root); @@ -40,9 +54,10 @@ export async function listReleaseFiles(root) { } export async function validateSelfContainedRelease(root) { + const resolvedRoot = path.resolve(root); const files = await listReleaseFiles(root); for (const file of files.filter((entry) => path.posix.basename(entry.relPath) === "package.json")) { - const packageDir = path.resolve(root, fromPosixRel(path.posix.dirname(file.relPath))); + const packageDir = path.resolve(resolvedRoot, fromPosixRel(path.posix.dirname(file.relPath))); const packageJson = JSON.parse(file.bytes.toString("utf8")); for (const section of PACKAGE_DEPENDENCY_SECTIONS) { const dependencies = packageJson[section]; @@ -51,7 +66,7 @@ export async function validateSelfContainedRelease(root) { for (const [name, spec] of Object.entries(dependencies)) { if (typeof spec !== "string" || !spec.startsWith("file:")) continue; const targetDir = path.resolve(packageDir, spec.slice(5)); - if (!isWithinRoot(root, targetDir)) { + if (!isWithinRoot(resolvedRoot, targetDir)) { throw new Error( `Release target is not self-contained: ${file.relPath} depends on ${name} via ${spec}`, ); @@ -61,9 +76,35 @@ export async function validateSelfContainedRelease(root) { }); } } + + for (const target of getPackageBinTargets(packageJson)) { + const targetPath = path.resolve(packageDir, target); + if (!isWithinRoot(resolvedRoot, targetPath)) { + throw new Error(`Release target is not self-contained: ${file.relPath} bin points to ${target}`); + } + await fs.access(targetPath).catch(() => { + throw new Error(`Missing package bin target for release: ${file.relPath} -> ${target}`); + }); + } } } +export function mimeType(relPath) { + const ext = path.extname(relPath).toLowerCase(); + return MIME_MAP[ext] || "text/plain"; +} + +function getPackageBinTargets(packageJson) { + const bin = packageJson.bin; + if (typeof bin === "string" && bin.trim()) { + return [bin.trim()]; + } + if (!bin || typeof bin !== "object" || Array.isArray(bin)) { + return []; + } + return Object.values(bin).filter((value) => typeof value === "string" && value.trim()); +} + function fromPosixRel(relPath) { return relPath === "." ? "." : relPath.split("/").join(path.sep); } diff --git a/scripts/lib/release-files.test.ts b/scripts/lib/release-files.test.ts index 898c93f..ead0bfd 100644 --- a/scripts/lib/release-files.test.ts +++ b/scripts/lib/release-files.test.ts @@ -65,6 +65,40 @@ test("validateSelfContainedRelease accepts file dependencies that stay within th await assert.doesNotReject(() => validateSelfContainedRelease(root)); }); +test("validateSelfContainedRelease accepts package bin targets inside the release root", async (t) => { + const root = await makeTempDir("baoyu-release-bin-ok-"); + t.after(() => fs.rm(root, { recursive: true, force: true })); + + await writeJson(path.join(root, "scripts", "package.json"), { + name: "test-skill-scripts", + version: "1.0.0", + bin: { + "test-skill": "./test-skill", + }, + }); + await writeFile(path.join(root, "scripts", "test-skill"), "#!/usr/bin/env sh\n"); + + await assert.doesNotReject(() => validateSelfContainedRelease(root)); +}); + +test("validateSelfContainedRelease rejects missing package bin targets", async (t) => { + const root = await makeTempDir("baoyu-release-bin-missing-"); + t.after(() => fs.rm(root, { recursive: true, force: true })); + + await writeJson(path.join(root, "scripts", "package.json"), { + name: "test-skill-scripts", + version: "1.0.0", + bin: { + "test-skill": "./missing", + }, + }); + + await assert.rejects( + () => validateSelfContainedRelease(root), + /Missing package bin target for release/, + ); +}); + test("validateSelfContainedRelease rejects missing local file dependencies", async (t) => { const root = await makeTempDir("baoyu-release-missing-"); t.after(() => fs.rm(root, { recursive: true, force: true })); diff --git a/scripts/publish-skill.mjs b/scripts/publish-skill.mjs index 07d43e3..2868e12 100644 --- a/scripts/publish-skill.mjs +++ b/scripts/publish-skill.mjs @@ -5,7 +5,7 @@ import { existsSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { listReleaseFiles, validateSelfContainedRelease } from "./lib/release-files.mjs"; +import { listReleaseFiles, mimeType, validateSelfContainedRelease } from "./lib/release-files.mjs"; const DEFAULT_REGISTRY = "https://clawhub.ai"; @@ -281,26 +281,6 @@ function titleCase(value) { .replace(/\b\w/g, (char) => char.toUpperCase()); } -const MIME_MAP = { - ".md": "text/markdown", - ".ts": "text/plain", - ".js": "text/javascript", - ".mjs": "text/javascript", - ".json": "application/json", - ".yml": "text/yaml", - ".yaml": "text/yaml", - ".txt": "text/plain", - ".html": "text/html", - ".css": "text/css", - ".xml": "text/xml", - ".svg": "image/svg+xml", -}; - -function mimeType(relPath) { - const ext = path.extname(relPath).toLowerCase(); - return MIME_MAP[ext] || "text/plain"; -} - function parseBoolean(value) { return ["1", "true", "yes", "on"].includes(String(value).trim().toLowerCase()); } diff --git a/scripts/sync-clawhub.mjs b/scripts/sync-clawhub.mjs index 801dd7f..a91df7d 100644 --- a/scripts/sync-clawhub.mjs +++ b/scripts/sync-clawhub.mjs @@ -6,47 +6,9 @@ import { existsSync } from "node:fs"; import path from "node:path"; import os from "node:os"; +import { listReleaseFiles, mimeType, validateSelfContainedRelease } from "./lib/release-files.mjs"; + const DEFAULT_REGISTRY = "https://clawhub.ai"; -const TEXT_EXTENSIONS = new Set([ - "md", - "mdx", - "txt", - "json", - "json5", - "yaml", - "yml", - "toml", - "js", - "cjs", - "mjs", - "ts", - "tsx", - "jsx", - "py", - "sh", - "rb", - "go", - "rs", - "swift", - "kt", - "java", - "cs", - "cpp", - "c", - "h", - "hpp", - "sql", - "csv", - "ini", - "cfg", - "env", - "xml", - "html", - "css", - "scss", - "sass", - "svg", -]); async function main() { const options = parseArgs(process.argv.slice(2)); @@ -75,7 +37,7 @@ async function main() { console.log(`Roots with skills: ${roots.join(", ")}`); const locals = await mapWithConcurrency(skills, options.concurrency, async (skill) => { - const files = await listTextFiles(skill.folder); + const files = await collectReleaseFiles(skill.folder); const fingerprint = buildFingerprint(files); return { ...skill, @@ -162,7 +124,7 @@ async function main() { console.log(`Publishing ${candidate.slug}@${version}`); try { - const files = await listTextFiles(candidate.folder); + const files = await collectReleaseFiles(candidate.folder); await publishSkill({ registry, token: config.token, @@ -363,35 +325,9 @@ async function hasSkillMarker(folder) { ); } -async function listTextFiles(root) { - const files = []; - - async function walk(folder) { - const entries = await fs.readdir(folder, { withFileTypes: true }); - for (const entry of entries) { - if (entry.name.startsWith(".")) continue; - if (entry.name === "node_modules") continue; - if (entry.name === ".clawhub" || entry.name === ".clawdhub") continue; - - const fullPath = path.join(folder, entry.name); - if (entry.isDirectory()) { - await walk(fullPath); - continue; - } - if (!entry.isFile()) continue; - - const relPath = path.relative(root, fullPath).split(path.sep).join("/"); - const ext = relPath.split(".").pop()?.toLowerCase() ?? ""; - if (!TEXT_EXTENSIONS.has(ext)) continue; - - const bytes = await fs.readFile(fullPath); - files.push({ relPath, bytes }); - } - } - - await walk(root); - files.sort((left, right) => left.relPath.localeCompare(right.relPath)); - return files; +async function collectReleaseFiles(root) { + await validateSelfContainedRelease(root); + return listReleaseFiles(root); } function buildFingerprint(files) { @@ -421,7 +357,7 @@ async function publishSkill({ registry, token, skill, files, version, changelog, ); for (const file of files) { - form.append("files", new Blob([file.bytes], { type: "text/plain" }), file.relPath); + form.append("files", new Blob([file.bytes], { type: mimeType(file.relPath) }), file.relPath); } const response = await fetch(`${registry}/api/v1/skills`, {