Local NLP & Small Language Models (SLMs): On-Device Summarization, Sentiment Analysis & Qwen Runbook

// Calculates memory footprint for in-browser SLMs based on parameter count and quantization
export function calculateSlmMemoryFootprint(
  parameterCountMillions: number,
  quantizationBits: 4 | 8 | 16 | 32 = 4
): {
  weightBufferMb: number;
  recommendedVramMb: number;
  isViableOnMobile: boolean;
} {
  // Convert bits per weight to bytes
  const bytesPerParam = quantizationBits / 8;
  const rawWeightsMb = Math.round((parameterCountMillions * 1e6 * bytesPerParam) / (1024 * 1024));

  // KV cache and context overhead (approx 35% buffer for 2048 token window)
  const contextBufferMb = Math.round(rawWeightsMb * 0.35);
  const recommendedVramMb = rawWeightsMb + contextBufferMb;

  return {
    weightBufferMb: rawWeightsMb,
    recommendedVramMb,
    isViableOnMobile: recommendedVramMb <= 800 // Mobile browsers typically enforce ~1GB tab heap limits
  };
}

Computes raw weight size and recommended VRAM overhead to assess whether an SLM can safely run on consumer mobile and desktop devices.

import { pipeline } from '@huggingface/transformers';

export async function summarizeTextLocally(
  sourceDocument: string,
  mode: 'brief' | 'detailed' = 'brief'
): Promise<string> {
  const summarizer = await pipeline(
    'summarization',
    'Xenova/distilbart-cnn-6-6',
    {
      device: 'webgpu',
      dtype: 'fp32'
    }
  );

  const isBrief = mode === 'brief';
  const result = await summarizer(sourceDocument, {
    max_length: isBrief ? 60 : 150,
    min_length: isBrief ? 20 : 50,
    length_penalty: isBrief ? 0.8 : 1.4,
    no_repeat_ngram_size: 3,
    num_beams: 2
  });

  return result[0]?.summary_text?.trim() || '';
}

Executes client-side abstractive document summarization using quantized DistilBART with configurable length and repetition penalties.

import { pipeline } from '@huggingface/transformers';

export interface SentenceSentiment {
  sentence: string;
  label: 'POSITIVE' | 'NEGATIVE';
  score: number;
}

export async function analyzeReviewSentiment(fullText: string): Promise<{
  overallScore: number;
  sentences: SentenceSentiment[];
}> {
  const classifier = await pipeline(
    'sentiment-analysis',
    'Xenova/distilbert-base-uncased-finetuned-sst-2-english',
    { device: 'webgpu', dtype: 'fp32' }
  );

  // Split into individual sentences
  const sentenceList = fullText
    .split(/(?<=[.?!])\s+/)
    .map((s) => s.trim())
    .filter((s) => s.length > 5);

  const sentences: SentenceSentiment[] = [];
  let positiveCount = 0;

  for (const sentence of sentenceList) {
    const [res] = await classifier(sentence);
    const label = res.label as 'POSITIVE' | 'NEGATIVE';
    const score = Math.round(res.score * 100);

    if (label === 'POSITIVE') positiveCount++;
    sentences.push({ sentence, label, score });
  }

  const overallScore = Math.round((positiveCount / Math.max(1, sentences.length)) * 100);
  return { overallScore, sentences };
}

Analyzes multi-sentence documents locally, generating individual emotional polarity scores and an aggregated sentiment score.

import { pipeline, TextStreamer } from '@huggingface/transformers';

export async function streamLocalChat(
  userPrompt: string,
  onToken: (tokenText: string) => void
): Promise<string> {
  const generator = await pipeline(
    'text-generation',
    'onnx-community/Qwen2.5-0.5B-Instruct',
    {
      device: 'webgpu',
      dtype: 'fp32'
    }
  );

  // Structured ChatML format
  const messages = [
    { role: 'system', content: 'You are a concise, helpful programming assistant.' },
    { role: 'user', content: userPrompt }
  ];

  // Initialize real-time streamer
  const streamer = new TextStreamer(generator.tokenizer, {
    skip_prompt: true,
    skip_special_tokens: true,
    callback_function: (token: string) => onToken(token)
  });

  const output = await generator(messages, {
    max_new_tokens: 256,
    temperature: 0.7,
    top_p: 0.9,
    streamer
  });

  return output[0]?.generated_text?.at(-1)?.content || '';
}

Streams generated tokens in real-time directly to the UI using Qwen2.5-0.5B and WebGPU hardware acceleration.

// Verify browser environment capabilities for running local Small Language Models
export async function auditSlmReadiness(): Promise<{
  webgpuSupported: boolean;
  maxComputeWorkgroupStorageSize: number;
  availableStorageMb: number;
}> {
  let webgpuSupported = false;
  let maxComputeWorkgroupStorageSize = 0;

  if ('gpu' in navigator) {
    const adapter = await navigator.gpu.requestAdapter();
    if (adapter) {
      webgpuSupported = true;
      maxComputeWorkgroupStorageSize = adapter.limits.maxComputeWorkgroupStorageSize;
    }
  }

  let availableStorageMb = 0;
  if ('storage' in navigator && 'estimate' in navigator.storage) {
    const estimate = await navigator.storage.estimate();
    availableStorageMb = Math.round(((estimate.quota || 0) - (estimate.usage || 0)) / (1024 * 1024));
  }

  return {
    webgpuSupported,
    maxComputeWorkgroupStorageSize,
    availableStorageMb
  };
}

Audits host GPU limits and persistent CacheStorage quota before initiating download of multi-hundred-megabyte SLM weights.