Client-Side Computer Vision & Neural Image Processing: Alpha Matting, ViT & Local Alt-Text Runbook

// Converts an HTMLImageElement or Canvas to a normalized planar NCHW Tensor
export function imageToNormalizedTensor(
  img: HTMLImageElement | HTMLCanvasElement,
  targetWidth = 512,
  targetHeight = 512
): { tensorData: Float32Array; dims: number[] } {
  const canvas = document.createElement('canvas');
  canvas.width = targetWidth;
  canvas.height = targetHeight;
  const ctx = canvas.getContext('2d', { willReadFrequently: true });
  if (!ctx) throw new Error('Failed to acquire 2D canvas context');

  ctx.drawImage(img, 0, 0, targetWidth, targetHeight);
  const { data } = ctx.getImageData(0, 0, targetWidth, targetHeight);

  // Allocate planar buffer for 3 channels (RGB)
  const numPixels = targetWidth * targetHeight;
  const tensorData = new Float32Array(3 * numPixels);

  // Standard ImageNet normalization coefficients
  const mean = [0.485, 0.456, 0.406];
  const std = [0.229, 0.224, 0.225];

  for (let i = 0; i < numPixels; i++) {
    const r = data[i * 4] / 255.0;
    const g = data[i * 4 + 1] / 255.0;
    const b = data[i * 4 + 2] / 255.0;

    // Planar layout: Channel 0 (R), Channel 1 (G), Channel 2 (B)
    tensorData[i] = (r - mean[0]) / std[0];
    tensorData[numPixels + i] = (g - mean[1]) / std[1];
    tensorData[2 * numPixels + i] = (b - mean[2]) / std[2];
  }

  return { tensorData, dims: [1, 3, targetHeight, targetWidth] };
}

Normalizes interleaved 8-bit RGBA canvas pixels into a planar [1, 3, H, W] 32-bit floating-point tensor compatible with ONNX vision models.

// Applies an estimated alpha matte to the original full-resolution image
export async function compositeTransparentImage(
  sourceImg: HTMLImageElement,
  alphaMatteData: Float32Array,
  matteWidth: number,
  matteHeight: number
): Promise<Blob> {
  const width = sourceImg.naturalWidth;
  const height = sourceImg.naturalHeight;

  // 1. Create matching offscreen canvas
  const canvas = document.createElement('canvas');
  canvas.width = width;
  canvas.height = height;
  const ctx = canvas.getContext('2d');
  if (!ctx) throw new Error('Failed to acquire canvas context');

  // 2. Render source image
  ctx.drawImage(sourceImg, 0, 0, width, height);
  const imgData = ctx.getImageData(0, 0, width, height);
  const pixels = imgData.data;

  // 3. Create intermediate canvas to upscale matte if model ran at fixed 512x512
  const matteCanvas = document.createElement('canvas');
  matteCanvas.width = matteWidth;
  matteCanvas.height = matteHeight;
  const matteCtx = matteCanvas.getContext('2d')!;
  const matteImgData = matteCtx.createImageData(matteWidth, matteHeight);

  for (let i = 0; i < alphaMatteData.length; i++) {
    const a = Math.round(Math.min(1, Math.max(0, alphaMatteData[i])) * 255);
    matteImgData.data[i * 4 + 3] = a; // alpha channel
  }
  matteCtx.putImageData(matteImgData, 0, 0);

  // 4. Sample upscaled matte onto target image alpha channel
  const upCanvas = document.createElement('canvas');
  upCanvas.width = width;
  upCanvas.height = height;
  const upCtx = upCanvas.getContext('2d')!;
  upCtx.drawImage(matteCanvas, 0, 0, width, height);
  const upMatte = upCtx.getImageData(0, 0, width, height).data;

  for (let i = 0; i < width * height; i++) {
    pixels[i * 4 + 3] = upMatte[i * 4 + 3];
  }
  ctx.putImageData(imgData, 0, 0);

  // 5. Export lossless transparent PNG
  return new Promise((resolve, reject) => {
    canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error('Export failed'))), 'image/png');
  });
}

Composites the continuous neural alpha channel over the original high-resolution image buffer, producing a clean transparent PNG.

import { pipeline } from '@huggingface/transformers';

export async function generateAccessibleAltText(
  imageSource: string | HTMLImageElement
): Promise<{ caption: string; htmlSnippet: string }> {
  // 1. Initialize vision encoder-decoder pipeline
  const captioner = await pipeline(
    'image-to-text',
    'Xenova/vit-gpt2-image-captioning',
    {
      device: 'webgpu', // Automatic fallback to WASM handled by runtime
      dtype: 'fp32'
    }
  );

  // 2. Generate natural language description
  const result = await captioner(imageSource, {
    max_new_tokens: 40,
    num_beams: 2
  });

  const rawCaption = result[0]?.generated_text?.trim() || 'Descriptive image';
  // Capitalize first letter and ensure ending period
  const formattedCaption = rawCaption.charAt(0).toUpperCase() + rawCaption.slice(1);
  const cleanAlt = formattedCaption.replace(/\.$/, '') + '.';

  // 3. Construct production WCAG-compliant HTML snippet
  const htmlSnippet = `<img src="image.webp" alt="${cleanAlt}" loading="lazy" decoding="async" />`;

  return { caption: cleanAlt, htmlSnippet };
}

Passes an image buffer into ViT-GPT2 to generate an accessible alt-text description without transmitting image pixels over any network.

export class VisionResourceManager {
  private activeUrls: Set<string> = new Set();
  private sharedCanvas: HTMLCanvasElement | null = null;

  // Reusable canvas to prevent DOM garbage thrashing
  public getCanvas(width: number, height: number): HTMLCanvasElement {
    if (!this.sharedCanvas) {
      this.sharedCanvas = document.createElement('canvas');
    }
    this.sharedCanvas.width = width;
    this.sharedCanvas.height = height;
    return this.sharedCanvas;
  }

  // Safe object URL generator with tracked lifecycle
  public createSafeUrl(blob: Blob): string {
    const url = URL.createObjectURL(blob);
    this.activeUrls.add(url);
    return url;
  }

  // Revoke single URL
  public revokeUrl(url: string) {
    if (this.activeUrls.has(url)) {
      URL.revokeObjectURL(url);
      this.activeUrls.delete(url);
    }
  }

  // Full teardown on route change or component unmount
  public dispose() {
    for (const url of this.activeUrls) {
      URL.revokeObjectURL(url);
    }
    this.activeUrls.clear();
    if (this.sharedCanvas) {
      this.sharedCanvas.width = 0;
      this.sharedCanvas.height = 0;
      this.sharedCanvas = null;
    }
  }
}

Resource lifecycle controller preventing memory leaks by tracking and revoking object URLs and recycling canvas buffers.

// Verify if the host GPU supports modern 16-bit float tensor operations
export async function checkGpuVisionFeatures(): Promise<{
  supportsWebGpu: boolean;
  supportsFp16: boolean;
  maxTextureDimension2D: number;
}> {
  if (!('gpu' in navigator)) {
    return { supportsWebGpu: false, supportsFp16: false, maxTextureDimension2D: 0 };
  }

  const adapter = await navigator.gpu.requestAdapter();
  if (!adapter) {
    return { supportsWebGpu: false, supportsFp16: false, maxTextureDimension2D: 0 };
  }

  return {
    supportsWebGpu: true,
    supportsFp16: adapter.features.has('shader-f16'),
    maxTextureDimension2D: adapter.limits.maxTextureDimension2D
  };
}

Inspects WebGPU adapter limits and checks for shader-f16 support to enable faster, half-precision vision inference.