feat(baoyu-image-gen): add OpenAI GPT Image edits support for reference images

- Support --ref with OpenAI GPT Image models (gpt-image-1.5)
- Auto-select Google or OpenAI when --ref provided
- Change ref-related warnings to explicit errors with fix hints
- Add reference image validation before generation
- Improve retry logic to skip non-retryable errors
This commit is contained in:
Jim Liu 宝玉
2026-02-06 16:06:58 -06:00
parent 7f80100b3e
commit 86a84739e8
5 changed files with 146 additions and 24 deletions
@@ -58,7 +58,9 @@ export async function generateImage(
if (!apiKey) throw new Error("DASHSCOPE_API_KEY is required");
if (args.referenceImages.length > 0) {
console.error("Warning: Reference images not yet supported with DashScope, ignoring.");
throw new Error(
"Reference images are not supported with DashScope provider in baoyu-image-gen. Use --provider google with a Gemini multimodal model."
);
}
const size = args.size ? normalizeSize(args.size) : getSizeFromAspectRatio(args.aspectRatio, args.quality);
@@ -216,13 +216,17 @@ export async function generateImage(
): Promise<Uint8Array> {
if (isGoogleImagen(model)) {
if (args.referenceImages.length > 0) {
console.error("Warning: Reference images not supported with Imagen models, ignoring.");
throw new Error(
"Reference images are not supported with Imagen models. Use gemini-3-pro-image-preview or gemini-3-flash-preview."
);
}
return generateWithImagen(prompt, model, args);
}
if (!isGoogleMultimodal(model) && args.referenceImages.length > 0) {
console.error("Warning: Reference images are only supported with Gemini multimodal models.");
throw new Error(
"Reference images are only supported with Gemini multimodal models. Use gemini-3-pro-image-preview or gemini-3-flash-preview."
);
}
return generateWithGemini(prompt, model, args);
@@ -1,9 +1,13 @@
import path from "node:path";
import { readFile } from "node:fs/promises";
import type { CliArgs } from "../types";
export function getDefaultModel(): string {
return process.env.OPENAI_IMAGE_MODEL || "gpt-image-1.5";
}
type OpenAIImageResponse = { data: Array<{ url?: string; b64_json?: string }> };
function parseAspectRatio(ar: string): { width: number; height: number } | null {
const match = ar.match(/^(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)$/);
if (!match) return null;
@@ -66,20 +70,32 @@ export async function generateImage(
if (!apiKey) throw new Error("OPENAI_API_KEY is required");
if (args.referenceImages.length > 0) {
console.error("Warning: Reference images not supported with OpenAI, ignoring.");
}
const size = args.size || getOpenAISize(model, args.aspectRatio, args.quality);
const body: Record<string, any> = {
model,
prompt,
size,
};
if (args.referenceImages.length > 0) {
if (model.includes("dall-e-2") || model.includes("dall-e-3")) {
throw new Error(
"Reference images with OpenAI in this skill require GPT Image models. Use --model gpt-image-1.5 (or another gpt-image model)."
);
}
return generateWithOpenAIEdits(baseURL, apiKey, prompt, model, size, args.referenceImages, args.quality);
}
return generateWithOpenAIGenerations(baseURL, apiKey, prompt, model, size, args.quality);
}
async function generateWithOpenAIGenerations(
baseURL: string,
apiKey: string,
prompt: string,
model: string,
size: string,
quality: CliArgs["quality"]
): Promise<Uint8Array> {
const body: Record<string, any> = { model, prompt, size };
if (model.includes("dall-e-3")) {
body.quality = args.quality === "2k" ? "hd" : "standard";
body.quality = quality === "2k" ? "hd" : "standard";
}
const res = await fetch(`${baseURL}/images/generations`, {
@@ -96,7 +112,62 @@ export async function generateImage(
throw new Error(`OpenAI API error: ${err}`);
}
const result = (await res.json()) as { data: Array<{ url?: string; b64_json?: string }> };
const result = (await res.json()) as OpenAIImageResponse;
return extractImageFromResponse(result);
}
async function generateWithOpenAIEdits(
baseURL: string,
apiKey: string,
prompt: string,
model: string,
size: string,
referenceImages: string[],
quality: CliArgs["quality"]
): Promise<Uint8Array> {
const form = new FormData();
form.append("model", model);
form.append("prompt", prompt);
form.append("size", size);
if (model.includes("gpt-image")) {
form.append("quality", quality === "2k" ? "high" : "medium");
}
for (const refPath of referenceImages) {
const bytes = await readFile(refPath);
const filename = path.basename(refPath);
const mimeType = getMimeType(filename);
const blob = new Blob([bytes], { type: mimeType });
form.append("image[]", blob, filename);
}
const res = await fetch(`${baseURL}/images/edits`, {
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
},
body: form,
});
if (!res.ok) {
const err = await res.text();
throw new Error(`OpenAI edits API error: ${err}`);
}
const result = (await res.json()) as OpenAIImageResponse;
return extractImageFromResponse(result);
}
function getMimeType(filename: string): string {
const ext = path.extname(filename).toLowerCase();
if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
if (ext === ".webp") return "image/webp";
if (ext === ".gif") return "image/gif";
return "image/png";
}
async function extractImageFromResponse(result: OpenAIImageResponse): Promise<Uint8Array> {
const img = result.data[0];
if (img?.b64_json) {