238 lines
8.6 KiB
TypeScript
238 lines
8.6 KiB
TypeScript
import type { LandmarkPoint } from './types';
|
||
|
||
/**
|
||
* Hand gesture detection — single source of truth for finger state utilities.
|
||
*
|
||
* Uses MediaPipe HandLandmarker's 21 landmarks per hand:
|
||
* 0: wrist
|
||
* 1-4: thumb (CMC, MCP, IP, TIP)
|
||
* 5-8: index (MCP, PIP, DIP, TIP)
|
||
* 9-12: middle (MCP, PIP, DIP, TIP)
|
||
* 13-16: ring (MCP, PIP, DIP, TIP)
|
||
* 17-20: pinky (MCP, PIP, DIP, TIP)
|
||
*
|
||
* All coordinates are normalized 0-1. Y increases downward (top=0, bottom=1).
|
||
*/
|
||
|
||
// =============================================================================
|
||
// Canonical landmark indices
|
||
// =============================================================================
|
||
|
||
/** MediaPipe Hand Landmarker 21-point landmark index map. */
|
||
export const HAND_LANDMARK_MAP = {
|
||
WRIST: 0,
|
||
THUMB: { CMC: 1, MCP: 2, IP: 3, TIP: 4 },
|
||
INDEX: { MCP: 5, PIP: 6, DIP: 7, TIP: 8 },
|
||
MIDDLE: { MCP: 9, PIP: 10, DIP: 11, TIP: 12 },
|
||
RING: { MCP: 13, PIP: 14, DIP: 15, TIP: 16 },
|
||
PINKY: { MCP: 17, PIP: 18, DIP: 19, TIP: 20 },
|
||
} as const;
|
||
|
||
/** Non-thumb finger names for extension/curl detection. */
|
||
export type FingerName = 'index' | 'middle' | 'ring' | 'pinky';
|
||
|
||
/** MCP/PIP/TIP index triples derived from canonical HAND_LANDMARK_MAP. */
|
||
const FINGER_INDICES: Record<FingerName, { mcp: number; pip: number; tip: number }> = {
|
||
index: { mcp: HAND_LANDMARK_MAP.INDEX.MCP, pip: HAND_LANDMARK_MAP.INDEX.PIP, tip: HAND_LANDMARK_MAP.INDEX.TIP },
|
||
middle: { mcp: HAND_LANDMARK_MAP.MIDDLE.MCP, pip: HAND_LANDMARK_MAP.MIDDLE.PIP, tip: HAND_LANDMARK_MAP.MIDDLE.TIP },
|
||
ring: { mcp: HAND_LANDMARK_MAP.RING.MCP, pip: HAND_LANDMARK_MAP.RING.PIP, tip: HAND_LANDMARK_MAP.RING.TIP },
|
||
pinky: { mcp: HAND_LANDMARK_MAP.PINKY.MCP, pip: HAND_LANDMARK_MAP.PINKY.PIP, tip: HAND_LANDMARK_MAP.PINKY.TIP },
|
||
};
|
||
|
||
// =============================================================================
|
||
// Constants
|
||
// =============================================================================
|
||
|
||
const DEFAULT_THRESHOLD = 0.02;
|
||
|
||
/**
|
||
* Minimum MCP→TIP distance (normalized 0-1 coords) for a finger to be
|
||
* considered extended. Prevents false positives when landmarks cluster in a
|
||
* fist (TIP curls above PIP but stays within ~3% of MCP).
|
||
*/
|
||
const MIN_EXTENSION_REACH = 0.04;
|
||
|
||
// =============================================================================
|
||
// Reusable calculations
|
||
// =============================================================================
|
||
|
||
/** 2D Euclidean distance between two landmarks. */
|
||
export function dist2d(a: LandmarkPoint, b: LandmarkPoint): number {
|
||
return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);
|
||
}
|
||
|
||
/**
|
||
* PIP→TIP vertical separation for a finger.
|
||
* Measures how clearly extended or curled a finger appears — used for
|
||
* gesture confidence scoring. Higher = more visually distinct pose.
|
||
*/
|
||
export function fingerClarity(landmarks: readonly LandmarkPoint[], finger: FingerName): number {
|
||
const { pip, tip } = FINGER_INDICES[finger];
|
||
return Math.abs(landmarks[pip].y - landmarks[tip].y);
|
||
}
|
||
|
||
/**
|
||
* Confidence score from the weakest finger clarity in a set.
|
||
* Shared formula: clamp(0.7 + minClarity × 3, max 0.95).
|
||
* Returns higher confidence when finger positions are unambiguous.
|
||
*/
|
||
export function clarityConfidence(clarities: number[]): number {
|
||
const worst = Math.min(...clarities);
|
||
return Math.min(0.95, 0.7 + worst * 3);
|
||
}
|
||
|
||
// =============================================================================
|
||
// Finger state detection
|
||
// =============================================================================
|
||
|
||
/**
|
||
* Check if a finger is extended using combined position + distance checks.
|
||
*
|
||
* Primary path requires BOTH:
|
||
* 1. Y-position: TIP above PIP (Y increases downward)
|
||
* 2. Distance guard: MCP→TIP > MIN_EXTENSION_REACH — confirms the finger
|
||
* actually reaches out (prevents fist false positives where TIP curls
|
||
* back above PIP but stays within ~3% of MCP)
|
||
*
|
||
* Rotation fallback: for hands rotated so Y-axis isn't informative, a strong
|
||
* distance ratio (2.0×) with reliable MCP→PIP baseline passes on its own.
|
||
*/
|
||
export function isFingerExtended(
|
||
landmarks: readonly LandmarkPoint[],
|
||
finger: FingerName,
|
||
threshold: number = DEFAULT_THRESHOLD,
|
||
): boolean {
|
||
const { mcp, pip, tip } = FINGER_INDICES[finger];
|
||
const mcpToTip = dist2d(landmarks[mcp], landmarks[tip]);
|
||
|
||
// Primary: TIP above PIP AND finger reaches out from MCP
|
||
if (landmarks[tip].y < landmarks[pip].y - threshold && mcpToTip > MIN_EXTENSION_REACH) {
|
||
return true;
|
||
}
|
||
|
||
// Rotation fallback: strong distance ratio when MCP→PIP baseline is reliable.
|
||
// Requires mcpToPip > 0.02 to avoid near-zero denominator instability.
|
||
const mcpToPip = dist2d(landmarks[mcp], landmarks[pip]);
|
||
if (mcpToPip > 0.02 && mcpToTip > mcpToPip * 2.0) {
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Check if a finger is curled.
|
||
*
|
||
* A finger is curled if ANY of:
|
||
* 1. TIP below PIP (Y check — standard upright hand)
|
||
* 2. TIP below MCP knuckle (catches fists where TIP wraps under)
|
||
*/
|
||
export function isFingerCurled(
|
||
landmarks: readonly LandmarkPoint[],
|
||
finger: FingerName,
|
||
threshold: number = DEFAULT_THRESHOLD,
|
||
): boolean {
|
||
const { mcp, pip, tip } = FINGER_INDICES[finger];
|
||
|
||
// Primary: TIP below PIP (Y increases downward)
|
||
if (landmarks[tip].y > landmarks[pip].y + threshold) return true;
|
||
|
||
// Deep curl: TIP below MCP knuckle (catches tight fists)
|
||
if (landmarks[tip].y > landmarks[mcp].y) return true;
|
||
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Check if a finger is deeply curled (TIP below MCP knuckle).
|
||
* Stricter than isFingerCurled: rejects extended fingers that appear barely
|
||
* curled due to camera angle or landmark noise.
|
||
*/
|
||
export function isFingerDeeplyCurled(
|
||
landmarks: readonly LandmarkPoint[],
|
||
finger: FingerName,
|
||
): boolean {
|
||
const { mcp, tip } = FINGER_INDICES[finger];
|
||
return landmarks[tip].y > landmarks[mcp].y;
|
||
}
|
||
|
||
/**
|
||
* Check if thumb is extended, handedness-aware.
|
||
* MediaPipe "Right" = user's LEFT hand → thumb extends RIGHT in image (TIP.x > MCP.x)
|
||
* MediaPipe "Left" = user's RIGHT hand → thumb extends LEFT in image (TIP.x < MCP.x)
|
||
*/
|
||
export function isThumbExtended(
|
||
landmarks: readonly LandmarkPoint[],
|
||
handedness: 'Left' | 'Right',
|
||
threshold: number = DEFAULT_THRESHOLD,
|
||
): boolean {
|
||
const thumbTip = landmarks[HAND_LANDMARK_MAP.THUMB.TIP];
|
||
const thumbMcp = landmarks[HAND_LANDMARK_MAP.THUMB.MCP];
|
||
|
||
if (handedness === 'Right') {
|
||
return thumbTip.x > thumbMcp.x + threshold;
|
||
}
|
||
return thumbTip.x < thumbMcp.x - threshold;
|
||
}
|
||
|
||
/**
|
||
* Euclidean distance between thumb TIP and index TIP.
|
||
* Used for OK sign circle detection.
|
||
*/
|
||
export function thumbIndexDistance(landmarks: readonly LandmarkPoint[]): number {
|
||
return dist2d(landmarks[HAND_LANDMARK_MAP.THUMB.TIP], landmarks[HAND_LANDMARK_MAP.INDEX.TIP]);
|
||
}
|
||
|
||
/**
|
||
* Check if hand wrist is near face (nose tip).
|
||
* Uses 2D Euclidean distance on normalized coordinates.
|
||
*/
|
||
export function isHandNearFace(
|
||
handLandmarks: readonly LandmarkPoint[],
|
||
faceLandmarks: readonly LandmarkPoint[],
|
||
maxDistance: number = 0.35,
|
||
): boolean {
|
||
return dist2d(handLandmarks[HAND_LANDMARK_MAP.WRIST], faceLandmarks[1]) < maxDistance;
|
||
}
|
||
|
||
// =============================================================================
|
||
// Finger overlap detection
|
||
// =============================================================================
|
||
|
||
/** All finger names including thumb (for overlap detection). */
|
||
export type AnyFingerName = 'thumb' | 'index' | 'middle' | 'ring' | 'pinky';
|
||
|
||
/** Fingertip landmark indices keyed by finger name. */
|
||
const TIP_INDICES: Record<AnyFingerName, number> = {
|
||
thumb: HAND_LANDMARK_MAP.THUMB.TIP,
|
||
index: HAND_LANDMARK_MAP.INDEX.TIP,
|
||
middle: HAND_LANDMARK_MAP.MIDDLE.TIP,
|
||
ring: HAND_LANDMARK_MAP.RING.TIP,
|
||
pinky: HAND_LANDMARK_MAP.PINKY.TIP,
|
||
};
|
||
|
||
/** Minimum 2D distance between adjacent fingertips before overlap is reported. */
|
||
const OVERLAP_THRESHOLD = 0.04;
|
||
|
||
/** Adjacent finger pairs to check for overlap. */
|
||
const ADJACENT_PAIRS: ReadonlyArray<[AnyFingerName, AnyFingerName]> = [
|
||
['thumb', 'index'], ['index', 'middle'], ['middle', 'ring'], ['ring', 'pinky'],
|
||
];
|
||
|
||
/**
|
||
* Detect overlapping fingertips from a single hand's landmarks.
|
||
* Returns pairs of adjacent fingers whose tips are closer than OVERLAP_THRESHOLD.
|
||
* When fingers overlap from the camera's perspective, MediaPipe misplaces their
|
||
* keypoints — this allows the UI to guide users ("Spread your fingers apart").
|
||
*/
|
||
export function detectFingerOverlap(
|
||
landmarks: readonly LandmarkPoint[],
|
||
): Array<[AnyFingerName, AnyFingerName]> {
|
||
const pairs: Array<[AnyFingerName, AnyFingerName]> = [];
|
||
for (const [a, b] of ADJACENT_PAIRS) {
|
||
if (dist2d(landmarks[TIP_INDICES[a]], landmarks[TIP_INDICES[b]]) < OVERLAP_THRESHOLD) {
|
||
pairs.push([a, b]);
|
||
}
|
||
}
|
||
return pairs;
|
||
}
|