Foundations Updated 2026-09 View as Markdown

ml5.js

ml5.js is a friendly JavaScript library for the browser that makes machine learning accessible to artists and creative coders.

ml5.js Tutorial#

ml5.js is a friendly JavaScript library for the browser that makes machine learning accessible to artists and creative coders. Built on top of TensorFlow.js, it provides high-level functions for common ML tasks.

What You'll Learn#

Image classification and object detection

Pose estimation and body tracking

Style transfer and image generation

Text analysis and sentiment detection

Sound classification and pitch detection

Installation#

You can include ml5.js in your project via a CDN:

javascript

<script src="https://unpkg.com/ml5@latest/dist/ml5.min.js"></script>

1. Understanding Pre-trained Models#

ml5.js specializes in making pre-trained models easy to use. Let's understand what this means:

javascript

<script>
function explainPretrainedModels() {
  const explanation = `
    <h4>Pre-trained Models in ml5.js:</h4>
    <ul>
      <li><strong>MobileNet:</strong> Trained on ImageNet (1000+ object classes)</li>
      <li><strong>PoseNet:</strong> Detects human poses and body keypoints</li>
      <li><strong>COCO-SSD:</strong> Object detection for 80 common objects</li>
      <li><strong>BodyPix:</strong> Person segmentation and body part detection</li>
      <li><strong>StyleTransfer:</strong> Artistic style transfer</li>
    </ul>
    <p><strong>Benefits:</strong></p>
    <ul>
      <li>No training required - models work immediately</li>
      <li>High accuracy from professional training</li>
      <li>Optimized for web performance</li>
      <li>Easy to integrate and use</li>
    </ul>
  `;
  
  document.getElementById('explanation-output').innerHTML = explanation;
}

explainPretrainedModels();
</script>
<div id="explanation-output"></div>

2. Image Classification with MobileNet#

Let's start with the classic example of classifying images using MobileNet.

javascript

<script>
// Global variables
let classifier;
let isModelLoaded = false;

async function initializeClassifier() {
  const statusDiv = document.getElementById('classifier-status');
  statusDiv.innerHTML = '<p>🔄 Loading MobileNet model...</p>';
  
  try {
    classifier = await ml5.imageClassifier('MobileNet');
    isModelLoaded = true;
    statusDiv.innerHTML = '<p>✅ Model loaded! Click buttons below to classify images.</p>';
    console.log('MobileNet model loaded successfully!');
  } catch (error) {
    statusDiv.innerHTML = '<p>❌ Error loading model. Please refresh and try again.</p>';
    console.error('Error loading model:', error);
  }
}

async function classifyImage(imageSrc, imageName) {
  if (!isModelLoaded) {
    alert('Model is still loading. Please wait...');
    return;
  }
  
  const img = document.getElementById('classify-img');
  const output = document.getElementById('classification-output');
  
  output.innerHTML = '<p>🔄 Loading and classifying image...</p>';
  
  img.onload = async () => {
    try {
      output.innerHTML = '<p>🧠 Analyzing image...</p>';
      
      const results = await classifier.classify(img);
      
      let resultHTML = `<h4>Classification Results for "${imageName}":</h4>`;
      resultHTML += '<div style="background: #f0f8ff; padding: 15px; border-radius: 8px; margin: 10px 0;">';
      resultHTML += '<ol>';
      
      results.slice(0, 3).forEach((result, index) => {
        const confidence = (result.confidence * 100).toFixed(1);
        const emoji = index === 0 ? '🥇' : (index === 1 ? '🥈' : '🥉');
        resultHTML += `<li>${emoji} <strong>${result.label}</strong> (${confidence}% confidence)</li>`;
      });
      
      resultHTML += '</ol></div>';
      
      output.innerHTML = resultHTML;
    } catch (error) {
      output.innerHTML = '<p>❌ Error classifying image. Please try again.</p>';
      console.error('Classification error:', error);
    }
  };
  
  img.onerror = () => {
    output.innerHTML = '<p>❌ Error loading image. Please try again.</p>';
  };
  
  img.src = imageSrc;
}

// Initialize the classifier when the page loads
initializeClassifier();
</script>

<div id="classifier-status"></div>

<div style="margin: 20px 0;">
  <h4>🖼️ Choose an image to classify:</h4>
  <div style="display: flex; gap: 10px; flex-wrap: wrap; margin: 10px 0;">
    <button onclick="classifyImage('https://images.unsplash.com/photo-1546598300-f5d62e0b7b9b?w=400&q=80', 'Golden Retriever')" 
            style="padding: 8px 16px; background: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer;">
      🐕 Dog
    </button>
    <button onclick="classifyImage('https://images.unsplash.com/photo-1495052384700-271d7c5f5e8c?w=400&q=80', 'Orange Cat')" 
            style="padding: 8px 16px; background: #FF9800; color: white; border: none; border-radius: 4px; cursor: pointer;">
      🐱 Cat
    </button>
    <button onclick="classifyImage('https://images.unsplash.com/photo-1503803548695-c2a7b4a5b875?w=400&q=80', 'Sports Car')" 
            style="padding: 8px 16px; background: #2196F3; color: white; border: none; border-radius: 4px; cursor: pointer;">
      🚗 Car
    </button>
    <button onclick="classifyImage('https://images.unsplash.com/photo-1541963463532-d68292c34d19?w=400&q=80', 'Flowers')" 
            style="padding: 8px 16px; background: #E91E63; color: white; border: none; border-radius: 4px; cursor: pointer;">
      🌸 Flowers
    </button>
  </div>
</div>

<div style="text-align: center; margin: 20px 0;">
  <img id="classify-img" crossorigin="anonymous" width="300" height="200" 
       style="object-fit: cover; border: 3px solid #ddd; border-radius: 8px; display: none;"/>
</div>

<div id="classification-output"></div>

<script>
// Show image when it loads
document.getElementById('classify-img').addEventListener('load', function() {
  this.style.display = 'block';
});
</script>

3. Pose Estimation with PoseNet#

PoseNet can detect human poses and track body keypoints in real-time. Let's explore this capability:

javascript

<script>
let poseNet;
let isPoseModelLoaded = false;

async function initializePoseNet() {
  const statusDiv = document.getElementById('pose-status');
  statusDiv.innerHTML = '<p>🔄 Loading PoseNet model...</p>';
  
  try {
    poseNet = await ml5.poseNet();
    isPoseModelLoaded = true;
    statusDiv.innerHTML = '<p>✅ PoseNet loaded! Click a button to analyze poses.</p>';
    console.log('PoseNet model loaded successfully!');
  } catch (error) {
    statusDiv.innerHTML = '<p>❌ Error loading PoseNet. Please refresh and try again.</p>';
    console.error('Error loading PoseNet:', error);
  }
}

async function analyzePose(imageSrc, imageName) {
  if (!isPoseModelLoaded) {
    alert('PoseNet is still loading. Please wait...');
    return;
  }
  
  const img = document.getElementById('pose-img');
  const canvas = document.getElementById('pose-canvas');
  const ctx = canvas.getContext('2d');
  const output = document.getElementById('pose-output');
  
  output.innerHTML = '<p>🔄 Loading image and detecting poses...</p>';
  
  img.onload = async () => {
    try {
      // Set canvas size to match image
      canvas.width = 400;
      canvas.height = 300;
      
      // Draw the image on canvas
      ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
      
      output.innerHTML = '<p>🧠 Analyzing pose keypoints...</p>';
      
      // Get pose estimation
      const poses = await poseNet.singlePose(img);
      
      if (poses.pose) {
        drawKeypoints(ctx, poses.pose.keypoints, canvas.width / img.naturalWidth, canvas.height / img.naturalHeight);
        displayPoseInfo(poses.pose, output, imageName);
      } else {
        output.innerHTML = '<p>❌ No pose detected in this image.</p>';
      }
      
    } catch (error) {
      output.innerHTML = '<p>❌ Error analyzing pose. Please try again.</p>';
      console.error('Pose analysis error:', error);
    }
  };
  
  img.src = imageSrc;
}

function drawKeypoints(ctx, keypoints, scaleX, scaleY) {
  // Draw keypoints
  ctx.fillStyle = '#FF0000';
  ctx.strokeStyle = '#00FF00';
  ctx.lineWidth = 2;
  
  keypoints.forEach(keypoint => {
    if (keypoint.score > 0.3) { // Only draw confident keypoints
      const x = keypoint.position.x * scaleX;
      const y = keypoint.position.y * scaleY;
      
      // Draw point
      ctx.beginPath();
      ctx.arc(x, y, 5, 0, 2 * Math.PI);
      ctx.fill();
      
      // Draw label
      ctx.fillStyle = '#FFFFFF';
      ctx.font = '12px Arial';
      ctx.fillText(keypoint.part, x + 8, y - 8);
      ctx.fillStyle = '#FF0000';
    }
  });
}

function displayPoseInfo(pose, output, imageName) {
  const keypoints = pose.keypoints.filter(kp => kp.score > 0.3);
  
  let resultHTML = `<h4>Pose Analysis for "${imageName}":</h4>`;
  resultHTML += `<p><strong>Confidence Score:</strong> ${(pose.score * 100).toFixed(1)}%</p>`;
  resultHTML += `<p><strong>Detected Keypoints:</strong> ${keypoints.length}/17</p>`;
  
  resultHTML += '<details style="margin: 10px 0;">';
  resultHTML += '<summary style="cursor: pointer; font-weight: bold;">View Keypoint Details</summary>';
  resultHTML += '<div style="background: #f9f9f9; padding: 10px; margin: 5px 0; border-radius: 4px;">';
  
  keypoints.forEach(keypoint => {
    resultHTML += `<p><strong>${keypoint.part}:</strong> (${keypoint.position.x.toFixed(0)}, ${keypoint.position.y.toFixed(0)}) - ${(keypoint.score * 100).toFixed(1)}% confidence</p>`;
  });
  
  resultHTML += '</div></details>';
  
  output.innerHTML = resultHTML;
}

// Initialize PoseNet
initializePoseNet();
</script>

<div id="pose-status"></div>

<div style="margin: 20px 0;">
  <h4>🕺 Choose a pose to analyze:</h4>
  <div style="display: flex; gap: 10px; flex-wrap: wrap; margin: 10px 0;">
    <button onclick="analyzePose('https://images.unsplash.com/photo-1571019613454-1cb2f99b2d8b?w=400&q=80', 'Yoga Pose')" 
            style="padding: 8px 16px; background: #9C27B0; color: white; border: none; border-radius: 4px; cursor: pointer;">
      🧘 Yoga
    </button>
    <button onclick="analyzePose('https://images.unsplash.com/photo-1571019613454-1cb2f99b2d8b?w=400&q=80', 'Dancing')" 
            style="padding: 8px 16px; background: #FF5722; color: white; border: none; border-radius: 4px; cursor: pointer;">
      💃 Dance
    </button>
    <button onclick="analyzePose('https://images.unsplash.com/photo-1544717297-fa95b6ee9643?w=400&q=80', 'Running')" 
            style="padding: 8px 16px; background: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer;">
      🏃 Running
    </button>
  </div>
</div>

<div style="text-align: center; margin: 20px 0;">
  <img id="pose-img" crossorigin="anonymous" width="400" height="300" style="display: none;"/>
  <canvas id="pose-canvas" style="border: 2px solid #333; border-radius: 8px;"></canvas>
</div>

<div id="pose-output"></div>

Exercise

Try it

<h5>1. Model Loading</h5>
    <ul>
      <li>Always handle loading states and errors</li>
      <li>Use async/await for cleaner code</li>
      <li>Load models once and reuse them</li>
    </ul>
    <pre>const classifier = await ml5.imageClassifier('MobileNet').catch(handleError);</pre>

<h5>2. Performance Optimization</h5>
    <ul>
      <li>Resize images before processing</li>
      <li>Use appropriate model versions (MobileNet vs ResNet)</li>
      <li>Consider using GPU acceleration when available</li>
    </ul>

<h5>3. User Experience</h5>
    <ul>
      <li>Show loading indicators</li>
      <li>Provide confidence scores</li>
      <li>Handle edge cases gracefully</li>
      <li>Add fallbacks for unsupported browsers</li>
    </ul>

<h5>4. Privacy Considerations</h5>
    <ul>
      <li>Process data locally when possible</li>
      <li>Inform users about camera/microphone access</li>
      <li>Don't store sensitive data unnecessarily</li>
    </ul>

<h5>5. Model Selection Guide</h5>
    <ul>
      <li><strong>MobileNet:</strong> General image classification</li>
      <li><strong>PoseNet:</strong> Human pose detection</li>
      <li><strong>COCO-SSD:</strong> Object detection</li>
      <li><strong>BodyPix:</strong> Person segmentation</li>
      <li><strong>SpeechCommands:</strong> Voice command recognition</li>
    </ul>
  `;

document.getElementById('practices-output').innerHTML = practices;
}

showBestPractices();
</script>
<div id="practices-output"></div>

ready

Solution

<script>
// Challenge: Build an "AI Assistant" that can:
// 1. Classify images when uploaded
// 2. Detect poses in photos
// 3. Analyze sentiment of text input
// 4. Respond to voice commands

class AIAssistant {
  constructor() {
    this.imageClassifier = null;
    this.poseNet = null;
    this.sentiment = null;
    this.soundClassifier = null;
    this.isInitialized = false;
  }

  async initialize() {
    // TODO: Load all models
    // TODO: Set up UI event handlers
    // TODO: Create a unified interface

    document.getElementById('assistant-output').innerHTML = 'TODO: Implement AI Assistant';
  }

  async processImage(file) {
    // TODO: Handle image upload and analysis
  }

  async processText(text) {
    // TODO: Analyze text sentiment and respond
  }

  async processVoice() {
    // TODO: Listen for voice commands
  }
}

const assistant = new AIAssistant();
// assistant.initialize(); // Uncomment to start the challenge
</script>

<div style="border: 2px dashed #ccc; padding: 20px; margin: 20px 0; text-align: center;">
  <h4>🤖 AI Assistant Challenge</h4>
  <p>Build a multi-modal AI application using the concepts you've learned!</p>
  <button onclick="assistant.initialize()" 
          style="padding: 12px 24px; background: #FF9800; color: white; border: none; border-radius: 6px; cursor: pointer;">
    Start Building
  </button>
</div>

<div id="assistant-output"></div>

Get the JavaScript agent pack

A battle-tested AGENTS.md, the review checklist, and the failure-mode cheat sheet for JavaScript. One email, then occasional updates when the tooling shifts. No course pitch.

Unsubscribe in one click. We never sell the list. Or just take the AGENTS.md now — no email needed.