Local In-Browser AI: WebGPU, Chrome Built-In AI & Client-Side Models Runbook

// Comprehensive client-side capability check for local AI acceleration
export async function detectAiCapabilities(): Promise<{
  webgpu: boolean;
  adapterName: string;
  maxBufferSizeMb: number;
  builtInAi: boolean;
}> {
  const result = {
    webgpu: false,
    adapterName: 'None (CPU fallback)',
    maxBufferSizeMb: 0,
    builtInAi: false
  };

  // 1. Detect WebGPU hardware acceleration
  if ('gpu' in navigator && typeof navigator.gpu.requestAdapter === 'function') {
    try {
      const adapter = await navigator.gpu.requestAdapter({
        powerPreference: 'high-performance'
      });
      if (adapter) {
        result.webgpu = true;
        const info = (await adapter.requestAdapterInfo?.()) || adapter.info;
        result.adapterName = [info?.vendor, info?.architecture, info?.device]
          .filter(Boolean)
          .join(' ') || 'Standard WebGPU Adapter';
        result.maxBufferSizeMb = Math.round(adapter.limits.maxBufferSize / (1024 * 1024));
      }
    } catch (e) {
      console.warn('WebGPU adapter request failed:', e);
    }
  }

  // 2. Detect Chrome Built-In AI (window.ai / window.model)
  if ('ai' in window && 'languageModel' in (window as any).ai) {
    result.builtInAi = true;
  }

  return result;
}

Evaluates whether the client environment supports hardware-accelerated WebGPU compute or Chrome Built-In AI, falling back gracefully to WebAssembly SIMD.

// Initialize WebGPU compute context with error boundaries
async function initWebGpuCompute() {
  if (!('gpu' in navigator)) {
    throw new Error('WebGPU is not supported in this browser. Try Chrome 113+, Edge, or Safari 18+.');
  }

  const adapter = await navigator.gpu.requestAdapter({
    powerPreference: 'high-performance'
  });
  if (!adapter) throw new Error('No compatible GPU adapter found.');

  // Request device with maximum available buffer limits for model weights
  const device = await adapter.requestDevice({
    requiredLimits: {
      maxStorageBufferBindingSize: adapter.limits.maxStorageBufferBindingSize,
      maxBufferSize: adapter.limits.maxBufferSize
    }
  });

  // Watch for device loss (e.g. driver update, GPU sleep, or memory pressure)
  device.lost.then((info) => {
    console.error('WebGPU Device was lost:', info.reason, info.message);
    if (info.reason !== 'destroyed') {
      // Re-initialize model pipeline or trigger WASM fallback
      window.dispatchEvent(new CustomEvent('webgpu-device-lost'));
    }
  });

  return { adapter, device };
}

Handles high-performance adapter selection, requests maximum storage buffer limits, and attaches automated recovery handlers for GPU device-loss.

# 1. Open Google Chrome Dev/Canary or Chrome 128+
# Navigate to:
chrome://flags/#prompt-api-for-gemini-nano
# Set to: "Enabled"

# 2. Enable optimization guide model download:
chrome://flags/#optimization-guide-on-device-model
# Set to: "Enabled BypassPrefRequirement"

# 3. Relaunch Chrome and verify download in:
chrome://components
# Check "Optimization Guide On Device Model" -> Click "Check for update"

Configuration flags required to unlock Chrome's experimental on-device Gemini Nano Prompt API in Chromium browsers.

// Query Chrome's built-in language model with streaming response
export async function promptLocalGemini(promptText: string, onChunk: (text: string) => void) {
  // Feature detection
  if (!('ai' in window) || !('languageModel' in (window as any).ai)) {
    throw new Error('Chrome Built-In AI (window.ai.languageModel) is not available.');
  }

  const ai = (window as any).ai;
  const capabilities = await ai.languageModel.capabilities();

  if (capabilities.available === 'no') {
    throw new Error('Local Gemini Nano model is not available or supported on this device.');
  }

  // Create an isolated conversational session
  const session = await ai.languageModel.create({
    systemPrompt: 'You are an ultra-fast, local technical assistant running in the browser.'
  });

  try {
    // Stream tokens in real-time as they are generated
    const stream = session.promptStreaming(promptText);
    for await (const chunk of stream) {
      onChunk(chunk);
    }
  } finally {
    // Destroy session when finished to free GPU memory
    session.destroy();
  }
}

Creates a local Gemini Nano session, streams generated tokens directly to the UI, and guarantees memory destruction upon completion.

import { pipeline } from '@huggingface/transformers';

export async function runLocalSpeechRecognition(audioFile: File): Promise<string> {
  // 1. Check for WebGPU availability
  let useGpu = false;
  try {
    const gpu = (navigator as any).gpu;
    useGpu = !!gpu && !!(await gpu.requestAdapter());
  } catch {
    useGpu = false;
  }

  // 2. Instantiate pipeline with quantized weights and WebGPU device
  const transcriber = await pipeline(
    'automatic-speech-recognition',
    'onnx-community/whisper-tiny.en',
    {
      device: useGpu ? 'webgpu' : 'wasm',
      // WebGPU: FP32 encoder + Q4 merged decoder; WASM: Q8
      dtype: useGpu ? { encoder_model: 'fp32', decoder_model_merged: 'q4' } : 'q8',
      progress_callback: (progressInfo: any) => {
        if (progressInfo.status === 'progress') {
          console.log(`Downloading ${progressInfo.file}: ${progressInfo.progress}%`);
        }
      }
    }
  );

  // 3. Decode audio file to 16kHz Float32Array in browser AudioContext
  const audioCtx = new AudioContext({ sampleRate: 16000 });
  const arrayBuffer = await audioFile.arrayBuffer();
  const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
  const audioData = audioBuffer.getChannelData(0);

  // 4. Execute 100% local speech-to-text inference
  const output = await transcriber(audioData);
  await audioCtx.close();

  return output.text;
}

Complete production pipeline executing local audio transcription directly on the user's GPU using Transformers.js and ONNX Runtime Web.

// Inspect total disk space used by cached AI models in browser CacheStorage
export async function getLocalModelStorageStats(cacheName = 'transformers-cache') {
  if (!('caches' in window)) return { totalBytes: 0, fileCount: 0 };

  const hasCache = await caches.has(cacheName);
  if (!hasCache) return { totalBytes: 0, fileCount: 0 };

  const cache = await caches.open(cacheName);
  const requests = await cache.keys();
  let totalBytes = 0;

  for (const req of requests) {
    const response = await cache.match(req);
    if (!response) continue;
    const len = Number(response.headers.get('content-length'));
    totalBytes += Number.isFinite(len) && len > 0 
      ? len 
      : (await response.clone().blob()).size;
  }

  return {
    totalBytes,
    fileCount: requests.length,
    formattedMb: (totalBytes / (1024 * 1024)).toFixed(1) + ' MB'
  };
}

Iterates through browser CacheStorage keys to compute accurate disk usage for client-side model repositories.

// Verify browser storage quota before caching model weights
export async function checkStorageQuota() {
  if ('storage' in navigator && 'estimate' in navigator.storage) {
    const { quota, usage } = await navigator.storage.estimate();
    const quotaMb = Math.round((quota || 0) / (1024 * 1024));
    const usageMb = Math.round((usage || 0) / (1024 * 1024));
    const remainingMb = quotaMb - usageMb;

    console.log(`Storage: ${usageMb} MB used of ${quotaMb} MB (${remainingMb} MB free)`);

    // Request persistent storage so the browser will not silently evict model weights
    if ('persist' in navigator.storage) {
      const isPersisted = await navigator.storage.persist();
      console.log('Persistent storage granted:', isPersisted);
    }
  }
}

Inspects remaining browser storage and requests storage persistence to protect model weights from automated browser cache eviction.