fix(coding-agent): use async operations in tools
This commit is contained in:
Mario Zechner
164
packages/coding-agent/src/utils/image-resize-core.ts
Normal file
164
packages/coding-agent/src/utils/image-resize-core.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
import { applyExifOrientation } from "./exif-orientation.ts";
|
||||
import { loadPhoton } from "./photon.ts";
|
||||
|
||||
export interface ImageResizeOptions {
|
||||
maxWidth?: number; // Default: 2000
|
||||
maxHeight?: number; // Default: 2000
|
||||
maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit)
|
||||
jpegQuality?: number; // Default: 80
|
||||
}
|
||||
|
||||
export interface ResizedImage {
|
||||
data: string; // base64
|
||||
mimeType: string;
|
||||
originalWidth: number;
|
||||
originalHeight: number;
|
||||
width: number;
|
||||
height: number;
|
||||
wasResized: boolean;
|
||||
}
|
||||
|
||||
// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit.
|
||||
const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
|
||||
|
||||
const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
|
||||
maxWidth: 2000,
|
||||
maxHeight: 2000,
|
||||
maxBytes: DEFAULT_MAX_BYTES,
|
||||
jpegQuality: 80,
|
||||
};
|
||||
|
||||
interface EncodedCandidate {
|
||||
data: string;
|
||||
encodedSize: number;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate {
|
||||
const data = Buffer.from(buffer).toString("base64");
|
||||
return {
|
||||
data,
|
||||
encodedSize: Buffer.byteLength(data, "utf-8"),
|
||||
mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize an image to fit within the specified max dimensions and encoded file size.
|
||||
* Returns null if the image cannot be resized below maxBytes.
|
||||
*
|
||||
* Uses Photon (Rust/WASM) for image processing. If Photon is not available,
|
||||
* returns null.
|
||||
*
|
||||
* Strategy for staying under maxBytes:
|
||||
* 1. First resize to maxWidth/maxHeight
|
||||
* 2. Try both PNG and JPEG formats, pick the smaller one
|
||||
* 3. If still too large, try JPEG with decreasing quality
|
||||
* 4. If still too large, progressively reduce dimensions until 1x1
|
||||
*/
|
||||
export async function resizeImageInProcess(
|
||||
inputBytes: Uint8Array,
|
||||
mimeType: string,
|
||||
options?: ImageResizeOptions,
|
||||
): Promise<ResizedImage | null> {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4;
|
||||
|
||||
const photon = await loadPhoton();
|
||||
if (!photon) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let image: ReturnType<typeof photon.PhotonImage.new_from_byteslice> | undefined;
|
||||
try {
|
||||
const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes);
|
||||
image = applyExifOrientation(photon, rawImage, inputBytes);
|
||||
if (image !== rawImage) rawImage.free();
|
||||
|
||||
const originalWidth = image.get_width();
|
||||
const originalHeight = image.get_height();
|
||||
const format = mimeType.split("/")[1] ?? "png";
|
||||
|
||||
// Check if already within all limits (dimensions AND encoded size)
|
||||
if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) {
|
||||
return {
|
||||
data: Buffer.from(inputBytes).toString("base64"),
|
||||
mimeType: mimeType || `image/${format}`,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
width: originalWidth,
|
||||
height: originalHeight,
|
||||
wasResized: false,
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate initial dimensions respecting max limits
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
|
||||
if (targetWidth > opts.maxWidth) {
|
||||
targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
|
||||
targetWidth = opts.maxWidth;
|
||||
}
|
||||
if (targetHeight > opts.maxHeight) {
|
||||
targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
|
||||
targetHeight = opts.maxHeight;
|
||||
}
|
||||
|
||||
function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] {
|
||||
const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3);
|
||||
|
||||
try {
|
||||
const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")];
|
||||
for (const quality of jpegQualities) {
|
||||
candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg"));
|
||||
}
|
||||
return candidates;
|
||||
} finally {
|
||||
resized.free();
|
||||
}
|
||||
}
|
||||
|
||||
const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40]));
|
||||
let currentWidth = targetWidth;
|
||||
let currentHeight = targetHeight;
|
||||
|
||||
while (true) {
|
||||
const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps);
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.encodedSize < opts.maxBytes) {
|
||||
return {
|
||||
data: candidate.data,
|
||||
mimeType: candidate.mimeType,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
width: currentWidth,
|
||||
height: currentHeight,
|
||||
wasResized: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (currentWidth === 1 && currentHeight === 1) {
|
||||
break;
|
||||
}
|
||||
|
||||
const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75));
|
||||
const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75));
|
||||
if (nextWidth === currentWidth && nextHeight === currentHeight) {
|
||||
break;
|
||||
}
|
||||
|
||||
currentWidth = nextWidth;
|
||||
currentHeight = nextHeight;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (image) {
|
||||
image.free();
|
||||
}
|
||||
}
|
||||
}
|
||||
42
packages/coding-agent/src/utils/image-resize-worker.ts
Normal file
42
packages/coding-agent/src/utils/image-resize-worker.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { parentPort } from "node:worker_threads";
|
||||
import { type ImageResizeOptions, type ResizedImage, resizeImageInProcess } from "./image-resize-core.ts";
|
||||
|
||||
interface ResizeImageWorkerRequest {
|
||||
inputBytes: Uint8Array;
|
||||
mimeType: string;
|
||||
options?: ImageResizeOptions;
|
||||
}
|
||||
|
||||
interface ResizeImageWorkerResponse {
|
||||
result?: ResizedImage | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function isResizeImageWorkerRequest(value: unknown): value is ResizeImageWorkerRequest {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const record = value as Record<string, unknown>;
|
||||
return record.inputBytes instanceof Uint8Array && typeof record.mimeType === "string";
|
||||
}
|
||||
|
||||
const port = parentPort;
|
||||
if (!port) {
|
||||
throw new Error("image resize worker requires parentPort");
|
||||
}
|
||||
|
||||
port.once("message", (message: unknown) => {
|
||||
void (async () => {
|
||||
try {
|
||||
if (!isResizeImageWorkerRequest(message)) {
|
||||
throw new Error("Invalid image resize worker request");
|
||||
}
|
||||
const result = await resizeImageInProcess(message.inputBytes, message.mimeType, message.options);
|
||||
const response: ResizeImageWorkerResponse = { result };
|
||||
port.postMessage(response);
|
||||
} catch (error) {
|
||||
const response: ResizeImageWorkerResponse = {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
port.postMessage(response);
|
||||
}
|
||||
})();
|
||||
});
|
||||
@@ -1,165 +1,96 @@
|
||||
import type { ImageContent } from "@earendil-works/pi-ai";
|
||||
import { applyExifOrientation } from "./exif-orientation.ts";
|
||||
import { loadPhoton } from "./photon.ts";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts";
|
||||
|
||||
export interface ImageResizeOptions {
|
||||
maxWidth?: number; // Default: 2000
|
||||
maxHeight?: number; // Default: 2000
|
||||
maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit)
|
||||
jpegQuality?: number; // Default: 80
|
||||
export type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts";
|
||||
|
||||
interface ResizeImageWorkerResponse {
|
||||
result?: ResizedImage | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ResizedImage {
|
||||
data: string; // base64
|
||||
mimeType: string;
|
||||
originalWidth: number;
|
||||
originalHeight: number;
|
||||
width: number;
|
||||
height: number;
|
||||
wasResized: boolean;
|
||||
function toTransferableBytes(input: Uint8Array): Uint8Array<ArrayBuffer> {
|
||||
// Transfer detaches the buffer, so transfer a worker-owned copy and leave the
|
||||
// caller's bytes intact.
|
||||
return new Uint8Array(input);
|
||||
}
|
||||
|
||||
// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit.
|
||||
const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
|
||||
|
||||
const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
|
||||
maxWidth: 2000,
|
||||
maxHeight: 2000,
|
||||
maxBytes: DEFAULT_MAX_BYTES,
|
||||
jpegQuality: 80,
|
||||
};
|
||||
|
||||
interface EncodedCandidate {
|
||||
data: string;
|
||||
encodedSize: number;
|
||||
mimeType: string;
|
||||
function isResizeImageWorkerResponse(value: unknown): value is ResizeImageWorkerResponse {
|
||||
return value !== null && typeof value === "object";
|
||||
}
|
||||
|
||||
function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate {
|
||||
const data = Buffer.from(buffer).toString("base64");
|
||||
return {
|
||||
data,
|
||||
encodedSize: Buffer.byteLength(data, "utf-8"),
|
||||
mimeType,
|
||||
};
|
||||
function createResizeWorker(): Worker {
|
||||
const isTypeScriptRuntime = import.meta.url.endsWith(".ts");
|
||||
const workerUrl = new URL(
|
||||
isTypeScriptRuntime ? "./image-resize-worker.ts" : "./image-resize-worker.js",
|
||||
import.meta.url,
|
||||
);
|
||||
return new Worker(workerUrl);
|
||||
}
|
||||
|
||||
async function resizeImageInWorker(
|
||||
inputBytes: Uint8Array,
|
||||
mimeType: string,
|
||||
options?: ImageResizeOptions,
|
||||
): Promise<ResizedImage | null> {
|
||||
const worker = createResizeWorker();
|
||||
try {
|
||||
const inputBytesForWorker = toTransferableBytes(inputBytes);
|
||||
return await new Promise<ResizedImage | null>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const settle = (result: ResizedImage | null): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(result);
|
||||
};
|
||||
const fail = (error: Error): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
reject(error);
|
||||
};
|
||||
|
||||
worker.once("message", (message: unknown) => {
|
||||
if (!isResizeImageWorkerResponse(message)) {
|
||||
fail(new Error("Invalid image resize worker response"));
|
||||
return;
|
||||
}
|
||||
if (message.error) {
|
||||
fail(new Error(message.error));
|
||||
return;
|
||||
}
|
||||
settle(message.result ?? null);
|
||||
});
|
||||
worker.once("error", fail);
|
||||
worker.once("exit", (code) => {
|
||||
if (!settled) {
|
||||
fail(new Error(`Image resize worker exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
worker.postMessage(
|
||||
{
|
||||
inputBytes: inputBytesForWorker,
|
||||
mimeType,
|
||||
options,
|
||||
},
|
||||
[inputBytesForWorker.buffer],
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize an image to fit within the specified max dimensions and encoded file size.
|
||||
* Returns null if the image cannot be resized below maxBytes.
|
||||
*
|
||||
* Uses Photon (Rust/WASM) for image processing. If Photon is not available,
|
||||
* returns null.
|
||||
*
|
||||
* Strategy for staying under maxBytes:
|
||||
* 1. First resize to maxWidth/maxHeight
|
||||
* 2. Try both PNG and JPEG formats, pick the smaller one
|
||||
* 3. If still too large, try JPEG with decreasing quality
|
||||
* 4. If still too large, progressively reduce dimensions until 1x1
|
||||
* Runs Photon in a worker thread so WASM decoding, resizing, and encoding do not
|
||||
* block the TUI event loop. Worker failures are propagated instead of retried on
|
||||
* the main thread.
|
||||
*/
|
||||
export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage | null> {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
const inputBuffer = Buffer.from(img.data, "base64");
|
||||
const inputBase64Size = Buffer.byteLength(img.data, "utf-8");
|
||||
|
||||
const photon = await loadPhoton();
|
||||
if (!photon) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let image: ReturnType<typeof photon.PhotonImage.new_from_byteslice> | undefined;
|
||||
try {
|
||||
const inputBytes = new Uint8Array(inputBuffer);
|
||||
const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes);
|
||||
image = applyExifOrientation(photon, rawImage, inputBytes);
|
||||
if (image !== rawImage) rawImage.free();
|
||||
|
||||
const originalWidth = image.get_width();
|
||||
const originalHeight = image.get_height();
|
||||
const format = img.mimeType?.split("/")[1] ?? "png";
|
||||
|
||||
// Check if already within all limits (dimensions AND encoded size)
|
||||
if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) {
|
||||
return {
|
||||
data: img.data,
|
||||
mimeType: img.mimeType ?? `image/${format}`,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
width: originalWidth,
|
||||
height: originalHeight,
|
||||
wasResized: false,
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate initial dimensions respecting max limits
|
||||
let targetWidth = originalWidth;
|
||||
let targetHeight = originalHeight;
|
||||
|
||||
if (targetWidth > opts.maxWidth) {
|
||||
targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
|
||||
targetWidth = opts.maxWidth;
|
||||
}
|
||||
if (targetHeight > opts.maxHeight) {
|
||||
targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
|
||||
targetHeight = opts.maxHeight;
|
||||
}
|
||||
|
||||
function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] {
|
||||
const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3);
|
||||
|
||||
try {
|
||||
const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")];
|
||||
for (const quality of jpegQualities) {
|
||||
candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg"));
|
||||
}
|
||||
return candidates;
|
||||
} finally {
|
||||
resized.free();
|
||||
}
|
||||
}
|
||||
|
||||
const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40]));
|
||||
let currentWidth = targetWidth;
|
||||
let currentHeight = targetHeight;
|
||||
|
||||
while (true) {
|
||||
const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps);
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.encodedSize < opts.maxBytes) {
|
||||
return {
|
||||
data: candidate.data,
|
||||
mimeType: candidate.mimeType,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
width: currentWidth,
|
||||
height: currentHeight,
|
||||
wasResized: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (currentWidth === 1 && currentHeight === 1) {
|
||||
break;
|
||||
}
|
||||
|
||||
const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75));
|
||||
const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75));
|
||||
if (nextWidth === currentWidth && nextHeight === currentHeight) {
|
||||
break;
|
||||
}
|
||||
|
||||
currentWidth = nextWidth;
|
||||
currentHeight = nextHeight;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (image) {
|
||||
image.free();
|
||||
}
|
||||
}
|
||||
export async function resizeImage(
|
||||
inputBytes: Uint8Array,
|
||||
mimeType: string,
|
||||
options?: ImageResizeOptions,
|
||||
): Promise<ResizedImage | null> {
|
||||
return resizeImageInWorker(inputBytes, mimeType, options);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user