memlnaut-nisps/playground/js/ui/hand-tracker.js

// HandTracker — MediaPipe hand tracking input for NISPS playground
// Extracts 14 derived features from right hand, gesture recognition from left hand

// MediaPipe landmark indices
const WRIST = 0;
const THUMB_CMC = 1, THUMB_MCP = 2, THUMB_IP = 3, THUMB_TIP = 4;
const INDEX_MCP = 5, INDEX_PIP = 6, INDEX_DIP = 7, INDEX_TIP = 8;
const MIDDLE_MCP = 9, MIDDLE_PIP = 10, MIDDLE_DIP = 11, MIDDLE_TIP = 12;
const RING_MCP = 13, RING_PIP = 14, RING_DIP = 15, RING_TIP = 16;
const PINKY_MCP = 17, PINKY_PIP = 18, PINKY_DIP = 19, PINKY_TIP = 20;

const FINGER_LANDMARKS = [
  [THUMB_CMC, THUMB_MCP, THUMB_IP, THUMB_TIP],
  [INDEX_MCP, INDEX_PIP, INDEX_DIP, INDEX_TIP],
  [MIDDLE_MCP, MIDDLE_PIP, MIDDLE_DIP, MIDDLE_TIP],
  [RING_MCP, RING_PIP, RING_DIP, RING_TIP],
  [PINKY_MCP, PINKY_PIP, PINKY_DIP, PINKY_TIP],
];

const FINGER_TIPS = [THUMB_TIP, INDEX_TIP, MIDDLE_TIP, RING_TIP, PINKY_TIP];
const FINGER_MCPS = [THUMB_MCP, INDEX_MCP, MIDDLE_MCP, RING_MCP, PINKY_MCP];

// Hand connections for skeleton drawing
const HAND_CONNECTIONS = [
  [0,1],[1,2],[2,3],[3,4],
  [0,5],[5,6],[6,7],[7,8],
  [0,9],[9,10],[10,11],[11,12],
  [0,13],[13,14],[14,15],[15,16],
  [0,17],[17,18],[18,19],[19,20],
  [5,9],[9,13],[13,17],
];

// Default tuning parameters (exported for dev panel)
export const HAND_TRACKER_DEFAULTS = {
  minHandDetectionConfidence: 0.5,
  minHandPresenceConfidence: 0.5,
  minTrackingConfidence: 0.5,
  smoothingFactor: 0.4,
  gestureHoldMs: 400,
  useWorldLandmarks: false,
};

export class HandTracker {
  /**
   * @param {Object} options
   * @param {function(number[])} options.onTrackingInput - called with 14 derived features [0,1]
   * @param {function('thumbsup'|'thumbsdown')} options.onGesture - called when gesture confirmed
   * @param {function(boolean)} [options.onConnectionChange] - called when tracking starts/stops
   * @param {HTMLVideoElement} options.videoElement - video element for camera feed
   * @param {HTMLCanvasElement} options.overlayCanvas - canvas for skeleton drawing
   */
  constructor(options = {}) {
    this.onTrackingInput = options.onTrackingInput || (() => {});
    this.onGesture = options.onGesture || (() => {});
    this.onConnectionChange = options.onConnectionChange || null;
    this.videoElement = options.videoElement;
    this.overlayCanvas = options.overlayCanvas;
    this.overlayCtx = this.overlayCanvas?.getContext('2d');

    this.active = false;
    this.features = new Array(14).fill(0.5);
    this._handLandmarker = null;
    this._stream = null;
    this._rafId = null;
    this._lastDetectTime = 0;
    this._minDetectInterval = 33; // ~30fps, will increase if slow

    // Tuning parameters (runtime-adjustable via setOptions)
    this.opts = { ...HAND_TRACKER_DEFAULTS };

    // Gesture state
    this._gestureCandidate = null; // 'thumbsup' | 'thumbsdown' | null
    this._gestureStartTime = 0;
    this._gestureProgress = 0; // 0-1 for UI
    this._lastGestureFired = 0;

    // Smoothing
    this._smoothedFeatures = new Array(14).fill(0.5);

    // Status
    this._trackingRight = false;
    this._trackingLeft = false;
  }

  /**
   * Update tuning parameters at runtime.
   * Confidence changes require re-creating the HandLandmarker (async).
   */
  async setOptions(patch) {
    const prev = { ...this.opts };
    Object.assign(this.opts, patch);

    // Check if MediaPipe confidence thresholds changed — requires re-init
    const confidenceChanged =
      prev.minHandDetectionConfidence !== this.opts.minHandDetectionConfidence ||
      prev.minHandPresenceConfidence !== this.opts.minHandPresenceConfidence ||
      prev.minTrackingConfidence !== this.opts.minTrackingConfidence;

    if (confidenceChanged && this._handLandmarker) {
      await this._handLandmarker.setOptions({
        minHandDetectionConfidence: this.opts.minHandDetectionConfidence,
        minHandPresenceConfidence: this.opts.minHandPresenceConfidence,
        minTrackingConfidence: this.opts.minTrackingConfidence,
      });
      console.log('[HandTracker] Updated confidence thresholds:', this.opts);
    }
  }

  get gestureProgress() { return this._gestureProgress; }
  get gestureCandidate() { return this._gestureCandidate; }
  get trackingRight() { return this._trackingRight; }
  get trackingLeft() { return this._trackingLeft; }

  async start() {
    if (this.active) return;

    try {
      // Request camera
      this._stream = await navigator.mediaDevices.getUserMedia({
        video: { facingMode: 'user', width: { ideal: 640 }, height: { ideal: 480 } }
      });
      this.videoElement.srcObject = this._stream;
      await this.videoElement.play();

      // Load MediaPipe (only once)
      if (!this._handLandmarker) {
        await this._initHandLandmarker();
      }

      this.active = true;
      if (this.onConnectionChange) this.onConnectionChange(true);
      this._detectLoop();
    } catch (e) {
      console.error('[HandTracker] Failed to start:', e);
      this.stop();
      throw e;
    }
  }

  stop() {
    this.active = false;
    if (this._rafId) {
      cancelAnimationFrame(this._rafId);
      this._rafId = null;
    }
    if (this._stream) {
      for (const track of this._stream.getTracks()) track.stop();
      this._stream = null;
    }
    this.videoElement.srcObject = null;
    this._trackingRight = false;
    this._trackingLeft = false;
    if (this.onConnectionChange) this.onConnectionChange(false);
  }

  destroy() {
    this.stop();
    if (this._handLandmarker) {
      this._handLandmarker.close();
      this._handLandmarker = null;
    }
  }

  async _initHandLandmarker() {
    // Dynamic import of MediaPipe vision tasks
    const vision = await import('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/vision_bundle.mjs');
    const { HandLandmarker, FilesetResolver } = vision;

    const wasmFileset = await FilesetResolver.forVisionTasks(
      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm'
    );

    this._handLandmarker = await HandLandmarker.createFromOptions(wasmFileset, {
      baseOptions: {
        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
        delegate: 'GPU',
      },
      runningMode: 'VIDEO',
      numHands: 2,
      minHandDetectionConfidence: this.opts.minHandDetectionConfidence,
      minHandPresenceConfidence: this.opts.minHandPresenceConfidence,
      minTrackingConfidence: this.opts.minTrackingConfidence,
    });
  }

  _detectLoop() {
    if (!this.active) return;

    const now = performance.now();
    if (now - this._lastDetectTime >= this._minDetectInterval) {
      const frameStart = now;

      if (this.videoElement.readyState >= 2 && this._handLandmarker) {
        const results = this._handLandmarker.detectForVideo(this.videoElement, now);
        this._processResults(results, now);
      }

      // Adaptive frame rate: slow down if detection is heavy, recover gradually
      const elapsed = performance.now() - frameStart;
      if (elapsed > 25) {
        this._minDetectInterval = 66; // drop to 15fps
      } else if (this._minDetectInterval > 33) {
        this._minDetectInterval = 33; // recover to 30fps
      }
    }

    this._rafId = requestAnimationFrame(() => this._detectLoop());
  }

  _processResults(results, now) {
    // Set canvas to a fixed size matching the PIP aspect ratio
    if (this.overlayCtx) {
      this.overlayCanvas.width = 360;
      this.overlayCanvas.height = 270;
      this._drawBackground();
    }

    let rightHand = null;
    let leftHand = null;
    let rightHandNorm = null; // always normalized (for drawing)
    let leftHandNorm = null;

    // Classify hands
    if (results.handednesses && results.landmarks) {
      // Choose landmark source: world (meters, hand-centric) or normalized (image-relative)
      const useWorld = this.opts.useWorldLandmarks && results.worldLandmarks;
      const lmSource = useWorld ? results.worldLandmarks : results.landmarks;

      for (let i = 0; i < results.handednesses.length; i++) {
        const handedness = results.handednesses[i][0];
        const landmarks = lmSource[i];
        const normLandmarks = results.landmarks[i]; // always keep normalized for drawing
        if (handedness.categoryName === 'Right') {
          rightHand = landmarks;
          rightHandNorm = normLandmarks;
        } else {
          leftHand = landmarks;
          leftHandNorm = normLandmarks;
        }
      }
    }

    // If only one hand detected, use it as tracking hand
    if (!rightHand && leftHand) {
      rightHand = leftHand;
      rightHandNorm = leftHandNorm;
      leftHand = null;
      leftHandNorm = null;
    }

    this._trackingRight = !!rightHand;
    this._trackingLeft = !!leftHand;

    // Extract features from tracking hand (right)
    if (rightHand) {
      const raw = this._extractFeatures(rightHand);
      // Smooth features
      const sf = this.opts.smoothingFactor;
      for (let i = 0; i < 14; i++) {
        this._smoothedFeatures[i] += (raw[i] - this._smoothedFeatures[i]) * sf;
        this.features[i] = this._smoothedFeatures[i];
      }
      this.onTrackingInput(this.features);
    }

    // Gesture recognition from left hand (uses normalized landmarks for finger counting)
    if (leftHandNorm) {
      this._processGesture(leftHandNorm, now);
    } else {
      this._gestureCandidate = null;
      this._gestureProgress = 0;
    }

    // Draw skeletons with zone awareness (always use normalized landmarks for drawing)
    // In canvas (pre-CSS-mirror) coords: right 1/3 = gesture zone, left 2/3 = tracking zone
    // After CSS scaleX(-1): left 1/3 = gesture, right 2/3 = tracking
    if (this.overlayCtx) {
      const w = this.overlayCanvas.width;
      const dividerX = w * (2 / 3);

      if (rightHandNorm) {
        const avgX = rightHandNorm[WRIST].x * w;
        const crossingZone = avgX > dividerX;
        this._drawSkeleton(rightHandNorm, '#ff6a00', crossingZone ? 0.25 : 0.9);
      }
      if (leftHandNorm) {
        const avgX = leftHandNorm[WRIST].x * w;
        const crossingZone = avgX < dividerX;
        this._drawSkeleton(leftHandNorm, '#00ccff', crossingZone ? 0.25 : 0.9);
      }
    }
  }

  _extractFeatures(landmarks) {
    const f = new Array(14);
    const isWorld = this.opts.useWorldLandmarks;

    // 0-1: Palm position X, Y
    // For world landmarks, x/y are in meters centered on hand — normalize differently
    if (isWorld) {
      // World coords: origin at hand center, range roughly ±0.1m
      f[0] = clamp01((landmarks[WRIST].x + 0.1) / 0.2);
      f[1] = clamp01((landmarks[WRIST].y + 0.1) / 0.2);
    } else {
      f[0] = 1.0 - landmarks[WRIST].x; // mirror X
      f[1] = landmarks[WRIST].y;
    }

    // 2-6: Finger curl (thumb through pinky)
    for (let fi = 0; fi < 5; fi++) {
      f[2 + fi] = this._fingerCurl(landmarks, fi);
    }

    // 7-10: Finger spread (4 adjacent pairs)
    for (let fi = 0; fi < 4; fi++) {
      f[7 + fi] = this._fingerSpread(landmarks, fi);
    }

    // 11: Hand roll (rotation around forward axis)
    const wrist = landmarks[WRIST];
    const middleMcp = landmarks[MIDDLE_MCP];
    const dx = middleMcp.x - wrist.x;
    const dy = middleMcp.y - wrist.y;
    const roll = (Math.atan2(dx, -dy) / Math.PI + 1) * 0.5;
    f[11] = clamp01(roll);

    // 12: Hand pitch (tilt forward/back from z-depth difference)
    const avgTipZ = (landmarks[INDEX_TIP].z + landmarks[MIDDLE_TIP].z + landmarks[RING_TIP].z) / 3;
    if (isWorld) {
      // World z is in meters — typical pitch range ~±0.05m
      f[12] = clamp01((wrist.z - avgTipZ + 0.05) / 0.1);
    } else {
      f[12] = clamp01((wrist.z - avgTipZ + 0.15) / 0.3);
    }

    // 13: Pinch distance (thumb tip to index tip)
    const pinch = dist3d(landmarks[THUMB_TIP], landmarks[INDEX_TIP]);
    if (isWorld) {
      // World pinch: range 0–0.15m typically
      f[13] = clamp01(1.0 - pinch / 0.15);
    } else {
      f[13] = clamp01(1.0 - pinch / 0.3);
    }

    return f;
  }

  _fingerCurl(landmarks, fingerIndex) {
    const joints = FINGER_LANDMARKS[fingerIndex];
    // Angle at PIP joint (middle joint)
    const a = landmarks[joints[0]]; // MCP/CMC
    const b = landmarks[joints[1]]; // MCP/PIP
    const c = landmarks[joints[2]]; // PIP/DIP
    const d = landmarks[joints[3]]; // DIP/TIP

    // Use angle between base→mid and mid→tip vectors
    const v1x = b.x - a.x, v1y = b.y - a.y, v1z = b.z - a.z;
    const v2x = d.x - b.x, v2y = d.y - b.y, v2z = d.z - b.z;

    const dot = v1x * v2x + v1y * v2y + v1z * v2z;
    const mag1 = Math.sqrt(v1x * v1x + v1y * v1y + v1z * v1z) || 0.001;
    const mag2 = Math.sqrt(v2x * v2x + v2y * v2y + v2z * v2z) || 0.001;

    const cosAngle = clamp(dot / (mag1 * mag2), -1, 1);
    const angle = Math.acos(cosAngle); // 0 = straight, PI = fully bent

    // Also consider distance from tip to MCP (more robust)
    const tipDist = dist3d(landmarks[joints[0]], landmarks[joints[3]]);
    const baseDist = dist3d(landmarks[joints[0]], landmarks[joints[2]]);
    const ratio = baseDist > 0.001 ? tipDist / (baseDist * 1.8) : 1;

    // Blend angle-based and distance-based curl
    const angleCurl = clamp01(1.0 - angle / Math.PI);
    const distCurl = clamp01(1.0 - ratio);

    return clamp01(angleCurl * 0.4 + distCurl * 0.6);
  }

  _fingerSpread(landmarks, pairIndex) {
    // Spread between adjacent finger tips
    const tip1 = landmarks[FINGER_TIPS[pairIndex]];
    const tip2 = landmarks[FINGER_TIPS[pairIndex + 1]];
    const mcp1 = landmarks[FINGER_MCPS[pairIndex]];
    const mcp2 = landmarks[FINGER_MCPS[pairIndex + 1]];

    // Direction vectors from MCP to tip
    const v1x = tip1.x - mcp1.x, v1y = tip1.y - mcp1.y;
    const v2x = tip2.x - mcp2.x, v2y = tip2.y - mcp2.y;

    const dot = v1x * v2x + v1y * v2y;
    const mag1 = Math.sqrt(v1x * v1x + v1y * v1y) || 0.001;
    const mag2 = Math.sqrt(v2x * v2x + v2y * v2y) || 0.001;

    const cosAngle = clamp(dot / (mag1 * mag2), -1, 1);
    const angle = Math.acos(cosAngle); // 0 = parallel, larger = more spread

    // Normalize: typical spread is 0-0.5 radians
    return clamp01(angle / 0.6);
  }

  _processGesture(landmarks, now) {
    const extended = this._countExtendedFingers(landmarks);

    let candidate = null;
    if (extended === 1) candidate = 'thumbsup';
    else if (extended === 2) candidate = 'thumbsdown';

    if (candidate !== this._gestureCandidate) {
      // New gesture or cleared
      this._gestureCandidate = candidate;
      this._gestureStartTime = now;
      this._gestureProgress = 0;
    } else if (candidate) {
      // Same gesture continuing
      const elapsed = now - this._gestureStartTime;
      this._gestureProgress = Math.min(elapsed / this.opts.gestureHoldMs, 1);

      if (this._gestureProgress >= 1 && now - this._lastGestureFired > 800) {
        // Fire gesture
        this.onGesture(candidate);
        this._lastGestureFired = now;
        this._gestureCandidate = null;
        this._gestureProgress = 0;
      }
    }
  }

  _countExtendedFingers(landmarks) {
    let count = 0;

    // Thumb: check if tip is far from palm center (different axis)
    const thumbExtended = dist3d(landmarks[THUMB_TIP], landmarks[THUMB_MCP]) >
                          dist3d(landmarks[THUMB_IP], landmarks[THUMB_MCP]) * 1.2;

    // Other fingers: tip should be farther from wrist than PIP
    for (let fi = 1; fi < 5; fi++) {
      const joints = FINGER_LANDMARKS[fi];
      const tipToWrist = dist3d(landmarks[joints[3]], landmarks[WRIST]);
      const pipToWrist = dist3d(landmarks[joints[1]], landmarks[WRIST]);
      if (tipToWrist > pipToWrist * 1.05) count++;
    }

    // Don't count thumb for gesture (only counting index, middle, ring, pinky)
    return count;
  }

  _drawBackground() {
    const ctx = this.overlayCtx;
    const w = this.overlayCanvas.width;
    const h = this.overlayCanvas.height;

    // Dark background
    ctx.fillStyle = '#0a0a0a';
    ctx.fillRect(0, 0, w, h);

    // Zone backgrounds (subtle tint)
    // In canvas coords (pre-CSS-mirror): left 2/3 = tracking (right hand), right 1/3 = gesture (left hand)
    const dividerX = w * (2 / 3);

    // Tracking zone — very subtle warm tint
    ctx.fillStyle = 'rgba(255, 106, 0, 0.03)';
    ctx.fillRect(0, 0, dividerX, h);

    // Gesture zone — very subtle cool tint
    ctx.fillStyle = 'rgba(0, 204, 255, 0.03)';
    ctx.fillRect(dividerX, 0, w - dividerX, h);

    // Dashed divider line
    ctx.strokeStyle = 'rgba(255, 255, 255, 0.15)';
    ctx.lineWidth = 1;
    ctx.setLineDash([4, 4]);
    ctx.beginPath();
    ctx.moveTo(dividerX, 0);
    ctx.lineTo(dividerX, h);
    ctx.stroke();
    ctx.setLineDash([]);

    // Zone labels (drawn in canvas coords, CSS mirror flips them)
    ctx.font = '9px monospace';
    ctx.textAlign = 'center';

    // Tracking label (left 2/3 of canvas → right 2/3 of display)
    ctx.fillStyle = 'rgba(255, 106, 0, 0.3)';
    ctx.fillText('TRACKING', dividerX / 2, 12);

    // Gesture label (right 1/3 of canvas → left 1/3 of display)
    ctx.fillStyle = 'rgba(0, 204, 255, 0.3)';
    ctx.fillText('GESTURE', dividerX + (w - dividerX) / 2, 12);

    ctx.textAlign = 'start'; // reset
  }

  _drawSkeleton(landmarks, color, opacity) {
    const ctx = this.overlayCtx;
    const w = this.overlayCanvas.width;
    const h = this.overlayCanvas.height;

    // Draw connections
    ctx.strokeStyle = color;
    ctx.lineWidth = 2;
    ctx.globalAlpha = opacity * 0.8;

    for (const [a, b] of HAND_CONNECTIONS) {
      const la = landmarks[a], lb = landmarks[b];
      ctx.beginPath();
      ctx.moveTo(la.x * w, la.y * h);
      ctx.lineTo(lb.x * w, lb.y * h);
      ctx.stroke();
    }

    // Draw landmarks
    ctx.fillStyle = color;
    ctx.globalAlpha = opacity;
    for (const lm of landmarks) {
      ctx.beginPath();
      ctx.arc(lm.x * w, lm.y * h, 3, 0, Math.PI * 2);
      ctx.fill();
    }

    ctx.globalAlpha = 1;
  }
}

// --- Utility ---
function clamp(v, min, max) { return Math.max(min, Math.min(max, v)); }
function clamp01(v) { return clamp(v, 0, 1); }
function dist3d(a, b) {
  const dx = a.x - b.x, dy = a.y - b.y, dz = (a.z || 0) - (b.z || 0);
  return Math.sqrt(dx * dx + dy * dy + dz * dz);
}