perf(tasks/image): WebGPU acceleration + skip dataUri round-trip + cheap input validation

The reactive image pipeline was spending 5+ seconds on a 7-task chain because
each task (a) re-encoded its output as a base64 dataUri to "match input
form", (b) ran its inner loop in pure JS on the main thread, and (c) ran the
default deep JSON-Schema validator over the multi-megabyte Uint8ClampedArray
on every reactive run.

Changes:

- @workglow/util/media:
  - Add `texture` source kind to `Image` so chained GPU tasks can pass a
    `GPUTexture` along without intermediate readback. `Image.fromTexture()`,
    `Image.isGpuSupported()`, and a browser-only `getTexture()` upload
    on demand and cache via WeakMap.
  - New `imageGpu.ts` (lazy device + pipeline cache, upload/download helpers)
    and `imageGpuOps.ts` (compute shaders for sepia, invert, grayscale,
    brightness, contrast, posterize, threshold, tint, transparency,
    pixelate, separable box blur, flip h/v).
  - Browser `getPixels` materializes texture/bitmap/canvas/videoFrame
    sources directly instead of throwing, and `getImageData` short-circuits
    OffscreenCanvas/ImageBitmap inputs.

- @workglow/tasks/image:
  - `produceImageOutput` no longer re-encodes a dataUri output — pixels flow
    between tasks as `ImageBinary`/`Image`. Display and other terminal
    consumers encode lazily on demand.
  - New `imageOpDispatcher.ts` routes each op to a GPU compute pipeline when
    available, falling back to a tuned CPU implementation that mirrors the
    prior in-task behaviour (separable blur with running sums, LUT-based
    posterize/contrast, etc.).
  - New `imageOps.ts` defines paired GPU+CPU implementations for each
    operation, used by every task.
  - New `ImageTaskBase` overrides `validateInput` to swap image-typed fields
    for a tiny stand-in before delegating, so the validator exercises every
    other field but skips the pixel-array walk.
  - All ImageXTask files refactored to thin shims that call the dispatcher.

Author: claude

packages/tasks/src/common.ts

@@ -37,6 +37,9 @@ export * from "./task/image/ImageResizeTask";
 export * from "./task/image/ImageRotateTask";
 export * from "./task/image/ImageSchemas";
 export * from "./task/image/ImageSepiaTask";
+export * from "./task/image/imageOpDispatcher";
+export * from "./task/image/imageOps";
+export * from "./task/image/ImageTaskBase";
 export * from "./task/image/imageTaskIo";
 export * from "./task/image/ImageTextTask";
 export * from "./task/image/ImageThresholdTask";

packages/tasks/src/task/image/ImageBlurTask.ts

@@ -7,13 +7,14 @@
 import {
   CreateWorkflow,
   IExecuteReactiveContext,
-  Task,
   TaskConfig,
   Workflow,
 } from "@workglow/task-graph";
 import { DataPortSchema } from "@workglow/util/schema";
 import { ImageBinaryOrDataUriSchema, ImageFromSchema } from "./ImageSchemas";
-import { produceImageOutput } from "./imageTaskIo";
+import { ImageTaskBase } from "./ImageTaskBase";
+import { runImageOp } from "./imageOpDispatcher";
+import { BLUR_OP, ensureImageGpuApi } from "./imageOps";
 
 const inputSchema = {
   type: "object",
@@ -48,7 +49,7 @@ export class ImageBlurTask<
   Input extends ImageBlurTaskInput = ImageBlurTaskInput,
   Output extends ImageBlurTaskOutput = ImageBlurTaskOutput,
   Config extends TaskConfig = TaskConfig,
-> extends Task<Input, Output, Config> {
+> extends ImageTaskBase<Input, Output, Config> {
   static override readonly type = "ImageBlurTask";
   static override readonly category = "Image";
   public static override title = "Blur Image";
@@ -67,58 +68,10 @@ export class ImageBlurTask<
     _output: Output,
     _context: IExecuteReactiveContext
   ): Promise<Output> {
-    const { radius = 1 } = input;
-    const image = await produceImageOutput(input.image, (img) => {
-      const { data: src, width, height, channels } = img;
-      const kernelSize = radius * 2 + 1;
-
-      // Horizontal pass
-      const tmp = new Uint8ClampedArray(src.length);
-      for (let y = 0; y < height; y++) {
-        for (let c = 0; c < channels; c++) {
-          let sum = 0;
-          // Initialize running sum for first pixel
-          for (let k = -radius; k <= radius; k++) {
-            const x = Math.max(0, Math.min(k, width - 1));
-            sum += src[(y * width + x) * channels + c];
-          }
-          tmp[y * width * channels + c] = (sum / kernelSize + 0.5) | 0;
-
-          // Slide the window across the row
-          for (let x = 1; x < width; x++) {
-            const addX = Math.min(x + radius, width - 1);
-            const removeX = Math.max(x - radius - 1, 0);
-            sum +=
-              src[(y * width + addX) * channels + c] - src[(y * width + removeX) * channels + c];
-            tmp[(y * width + x) * channels + c] = (sum / kernelSize + 0.5) | 0;
-          }
-        }
-      }
-
-      // Vertical pass
-      const dst = new Uint8ClampedArray(src.length);
-      for (let x = 0; x < width; x++) {
-        for (let c = 0; c < channels; c++) {
-          let sum = 0;
-          for (let k = -radius; k <= radius; k++) {
-            const y = Math.max(0, Math.min(k, height - 1));
-            sum += tmp[(y * width + x) * channels + c];
-          }
-          dst[x * channels + c] = (sum / kernelSize + 0.5) | 0;
-
-          for (let y = 1; y < height; y++) {
-            const addY = Math.min(y + radius, height - 1);
-            const removeY = Math.max(y - radius - 1, 0);
-            sum +=
-              tmp[(addY * width + x) * channels + c] - tmp[(removeY * width + x) * channels + c];
-            dst[(y * width + x) * channels + c] = (sum / kernelSize + 0.5) | 0;
-          }
-        }
-      }
-
-      return { data: dst, width, height, channels };
-    });
-    return { image } as Output;
+    await ensureImageGpuApi();
+    const radius = input.radius ?? 1;
+    const image = await runImageOp(input.image, BLUR_OP, { radius });
+    return { image: image as unknown as Output["image"] } as Output;
   }
 }
 

packages/tasks/src/task/image/ImageBorderTask.ts

@@ -7,14 +7,15 @@
 import {
   CreateWorkflow,
   IExecuteReactiveContext,
-  Task,
   TaskConfig,
   Workflow,
 } from "@workglow/task-graph";
 import { resolveColor } from "@workglow/util/media";
 import { DataPortSchema } from "@workglow/util/schema";
 import { ColorValueSchema, ImageBinaryOrDataUriSchema, ImageFromSchema } from "./ImageSchemas";
-import { produceImageOutput } from "./imageTaskIo";
+import { ImageTaskBase } from "./ImageTaskBase";
+import { runImageResizeOp } from "./imageOpDispatcher";
+import { BORDER_OP } from "./imageOps";
 
 const inputSchema = {
   type: "object",
@@ -49,7 +50,7 @@ export class ImageBorderTask<
   Input extends ImageBorderTaskInput = ImageBorderTaskInput,
   Output extends ImageBorderTaskOutput = ImageBorderTaskOutput,
   Config extends TaskConfig = TaskConfig,
-> extends Task<Input, Output, Config> {
+> extends ImageTaskBase<Input, Output, Config> {
   static override readonly type = "ImageBorderTask";
   static override readonly category = "Image";
   public static override title = "Add Border";
@@ -68,43 +69,16 @@ export class ImageBorderTask<
     _output: Output,
     _context: IExecuteReactiveContext
   ): Promise<Output> {
-    const { borderWidth: bw = 1 } = input;
-    const color = resolveColor(input.color);
-    const image = await produceImageOutput(input.image, (img) => {
-      const { data: src, width: srcW, height: srcH, channels: srcCh } = img;
-      const outCh = 4;
-      const dstW = srcW + bw * 2;
-      const dstH = srcH + bw * 2;
-      const dst = new Uint8ClampedArray(dstW * dstH * outCh);
-
-      const r = color.r;
-      const g = color.g;
-      const b = color.b;
-      const a = color.a;
-
-      // Fill entire image with border color
-      for (let i = 0; i < dst.length; i += outCh) {
-        dst[i] = r;
-        dst[i + 1] = g;
-        dst[i + 2] = b;
-        dst[i + 3] = a;
-      }
-
-      // Copy source image into center
-      for (let y = 0; y < srcH; y++) {
-        for (let x = 0; x < srcW; x++) {
-          const srcIdx = (y * srcW + x) * srcCh;
-          const dstIdx = ((y + bw) * dstW + (x + bw)) * outCh;
-          dst[dstIdx] = src[srcIdx];
-          dst[dstIdx + 1] = srcCh >= 3 ? src[srcIdx + 1] : src[srcIdx];
-          dst[dstIdx + 2] = srcCh >= 3 ? src[srcIdx + 2] : src[srcIdx];
-          dst[dstIdx + 3] = srcCh === 4 ? src[srcIdx + 3] : 255;
-        }
-      }
-
-      return { data: dst, width: dstW, height: dstH, channels: outCh };
+    const { r, g, b, a } = resolveColor(input.color);
+    const borderWidth = input.borderWidth ?? 1;
+    const image = await runImageResizeOp(input.image, BORDER_OP, {
+      borderWidth,
+      r,
+      g,
+      b,
+      a,
     });
-    return { image } as Output;
+    return { image: image as unknown as Output["image"] } as Output;
   }
 }
 

packages/tasks/src/task/image/ImageBrightnessTask.ts

@@ -7,13 +7,14 @@
 import {
   CreateWorkflow,
   IExecuteReactiveContext,
-  Task,
   TaskConfig,
   Workflow,
 } from "@workglow/task-graph";
 import { DataPortSchema } from "@workglow/util/schema";
 import { ImageBinaryOrDataUriSchema, ImageFromSchema } from "./ImageSchemas";
-import { produceImageOutput } from "./imageTaskIo";
+import { ImageTaskBase } from "./ImageTaskBase";
+import { runImageOp } from "./imageOpDispatcher";
+import { BRIGHTNESS_OP, ensureImageGpuApi } from "./imageOps";
 
 const inputSchema = {
   type: "object",
@@ -48,7 +49,7 @@ export class ImageBrightnessTask<
   Input extends ImageBrightnessTaskInput = ImageBrightnessTaskInput,
   Output extends ImageBrightnessTaskOutput = ImageBrightnessTaskOutput,
   Config extends TaskConfig = TaskConfig,
-> extends Task<Input, Output, Config> {
+> extends ImageTaskBase<Input, Output, Config> {
   static override readonly type = "ImageBrightnessTask";
   static override readonly category = "Image";
   public static override title = "Adjust Brightness";
@@ -67,27 +68,10 @@ export class ImageBrightnessTask<
     _output: Output,
     _context: IExecuteReactiveContext
   ): Promise<Output> {
+    await ensureImageGpuApi();
     const amount = input.amount ?? 0;
-    const image = await produceImageOutput(input.image, (img) => {
-      const { data: src, width, height, channels } = img;
-      const dst = new Uint8ClampedArray(src.length);
-
-      if (channels === 4) {
-        for (let i = 0; i < src.length; i += 4) {
-          dst[i] = src[i]! + amount;
-          dst[i + 1] = src[i + 1]! + amount;
-          dst[i + 2] = src[i + 2]! + amount;
-          dst[i + 3] = src[i + 3]!; // preserve alpha
-        }
-      } else {
-        for (let i = 0; i < src.length; i++) {
-          dst[i] = src[i]! + amount;
-        }
-      }
-
-      return { data: dst, width, height, channels };
-    });
-    return { image } as Output;
+    const image = await runImageOp(input.image, BRIGHTNESS_OP, { amount });
+    return { image: image as unknown as Output["image"] } as Output;
   }
 }
 

packages/tasks/src/task/image/ImageContrastTask.ts

@@ -7,13 +7,14 @@
 import {
   CreateWorkflow,
   IExecuteReactiveContext,
-  Task,
   TaskConfig,
   Workflow,
 } from "@workglow/task-graph";
 import { DataPortSchema } from "@workglow/util/schema";
 import { ImageBinaryOrDataUriSchema, ImageFromSchema } from "./ImageSchemas";
-import { produceImageOutput } from "./imageTaskIo";
+import { ImageTaskBase } from "./ImageTaskBase";
+import { runImageOp } from "./imageOpDispatcher";
+import { CONTRAST_OP, ensureImageGpuApi } from "./imageOps";
 
 const inputSchema = {
   type: "object",
@@ -48,7 +49,7 @@ export class ImageContrastTask<
   Input extends ImageContrastTaskInput = ImageContrastTaskInput,
   Output extends ImageContrastTaskOutput = ImageContrastTaskOutput,
   Config extends TaskConfig = TaskConfig,
-> extends Task<Input, Output, Config> {
+> extends ImageTaskBase<Input, Output, Config> {
   static override readonly type = "ImageContrastTask";
   static override readonly category = "Image";
   public static override title = "Adjust Contrast";
@@ -67,35 +68,10 @@ export class ImageContrastTask<
     _output: Output,
     _context: IExecuteReactiveContext
   ): Promise<Output> {
+    await ensureImageGpuApi();
     const amount = input.amount ?? 0;
-    const image = await produceImageOutput(input.image, (img) => {
-      const { data: src, width, height, channels } = img;
-
-      // Precompute 256-entry lookup table
-      const factor = (259 * (amount + 255)) / (255 * (259 - amount));
-      const lut = new Uint8ClampedArray(256);
-      for (let i = 0; i < 256; i++) {
-        lut[i] = factor * (i - 128) + 128;
-      }
-
-      const dst = new Uint8ClampedArray(src.length);
-
-      if (channels === 4) {
-        for (let i = 0; i < src.length; i += 4) {
-          dst[i] = lut[src[i]!]!;
-          dst[i + 1] = lut[src[i + 1]!]!;
-          dst[i + 2] = lut[src[i + 2]!]!;
-          dst[i + 3] = src[i + 3]!; // preserve alpha
-        }
-      } else {
-        for (let i = 0; i < src.length; i++) {
-          dst[i] = lut[src[i]!]!;
-        }
-      }
-
-      return { data: dst, width, height, channels };
-    });
-    return { image } as Output;
+    const image = await runImageOp(input.image, CONTRAST_OP, { amount });
+    return { image: image as unknown as Output["image"] } as Output;
   }
 }
 

packages/tasks/src/task/image/ImageCropTask.ts

@@ -7,13 +7,14 @@
 import {
   CreateWorkflow,
   IExecuteReactiveContext,
-  Task,
   TaskConfig,
   Workflow,
 } from "@workglow/task-graph";
 import { DataPortSchema } from "@workglow/util/schema";
 import { ImageBinaryOrDataUriSchema, ImageFromSchema } from "./ImageSchemas";
-import { produceImageOutput } from "./imageTaskIo";
+import { ImageTaskBase } from "./ImageTaskBase";
+import { runImageResizeOp } from "./imageOpDispatcher";
+import { CROP_OP } from "./imageOps";
 
 const inputSchema = {
   type: "object",
@@ -44,7 +45,7 @@ export class ImageCropTask<
   Input extends ImageCropTaskInput = ImageCropTaskInput,
   Output extends ImageCropTaskOutput = ImageCropTaskOutput,
   Config extends TaskConfig = TaskConfig,
-> extends Task<Input, Output, Config> {
+> extends ImageTaskBase<Input, Output, Config> {
   static override readonly type = "ImageCropTask";
   static override readonly category = "Image";
   public static override title = "Crop Image";
@@ -63,35 +64,13 @@ export class ImageCropTask<
     _output: Output,
     _context: IExecuteReactiveContext
   ): Promise<Output> {
-    const { x: rawX, y: rawY, width: rawW, height: rawH } = input;
-    const image = await produceImageOutput(input.image, (img) => {
-      const { data: src, width: srcW, height: srcH, channels } = img;
-
-      if (srcW < 1 || srcH < 1) {
-        throw new RangeError("Cannot crop an empty image");
-      }
-
-      if (rawX < 0 || rawX >= srcW || rawY < 0 || rawY >= srcH) {
-        throw new RangeError("Crop origin is outside the source image bounds");
-      }
-
-      const x = rawX;
-      const y = rawY;
-      const w = Math.min(rawW, srcW - x);
-      const h = Math.min(rawH, srcH - y);
-
-      const dst = new Uint8ClampedArray(w * h * channels);
-      const rowBytes = w * channels;
-
-      for (let row = 0; row < h; row++) {
-        const srcOffset = ((y + row) * srcW + x) * channels;
-        const dstOffset = row * rowBytes;
-        dst.set(src.subarray(srcOffset, srcOffset + rowBytes), dstOffset);
-      }
-
-      return { data: dst, width: w, height: h, channels };
+    const image = await runImageResizeOp(input.image, CROP_OP, {
+      x: input.x,
+      y: input.y,
+      width: input.width,
+      height: input.height,
     });
-    return { image } as Output;
+    return { image: image as unknown as Output["image"] } as Output;
   }
 }