{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "objc[24504]: Class CaptureDelegate is implemented in both /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/mediapipe/.dylibs/libopencv_videoio.3.4.16.dylib (0x1153c8860) and /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/cv2/cv2.abi3.so (0x2876f6480). One of the two will be used. Which one is undefined.\n", "objc[24504]: Class CVWindow is implemented in both /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/mediapipe/.dylibs/libopencv_highgui.3.4.16.dylib (0x115110a68) and /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/cv2/cv2.abi3.so (0x2876f64d0). One of the two will be used. Which one is undefined.\n", "objc[24504]: Class CVView is implemented in both /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/mediapipe/.dylibs/libopencv_highgui.3.4.16.dylib (0x115110a90) and /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/cv2/cv2.abi3.so (0x2876f64f8). One of the two will be used. Which one is undefined.\n", "objc[24504]: Class CVSlider is implemented in both /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/mediapipe/.dylibs/libopencv_highgui.3.4.16.dylib (0x115110ab8) and /Users/fuixlabsdev1/Programming/PP/graduation-thesis/env/lib/python3.8/site-packages/cv2/cv2.abi3.so (0x2876f6520). One of the two will be used. Which one is undefined.\n" ] } ], "source": [ "import mediapipe as mp\n", "import cv2\n", "import numpy as np\n", "import pandas as pd\n", "import pickle\n", "\n", "# Drawing helpers\n", "mp_drawing = mp.solutions.drawing_utils\n", "mp_pose = mp.solutions.pose" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 1. Set up important functions and variables" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "IMPORTANT_LMS = [\n", " \"NOSE\",\n", " \"LEFT_SHOULDER\",\n", " \"RIGHT_SHOULDER\",\n", " \"LEFT_HIP\",\n", " \"RIGHT_HIP\",\n", " \"LEFT_KNEE\",\n", " \"RIGHT_KNEE\",\n", " \"LEFT_ANKLE\",\n", " \"RIGHT_ANKLE\"\n", "]\n", "\n", "headers = [\"label\"] # Label column\n", "\n", "for lm in IMPORTANT_LMS:\n", " headers += [f\"{lm.lower()}_x\", f\"{lm.lower()}_y\", f\"{lm.lower()}_z\", f\"{lm.lower()}_v\"]\n", "\n", "\n", "def extract_important_keypoints(results) -> list:\n", " '''\n", " Extract important keypoints from mediapipe pose detection\n", " '''\n", " landmarks = results.pose_landmarks.landmark\n", "\n", " data = []\n", " for lm in IMPORTANT_LMS:\n", " keypoint = landmarks[mp_pose.PoseLandmark[lm].value]\n", " data.append([keypoint.x, keypoint.y, keypoint.z, keypoint.visibility])\n", " \n", " return np.array(data).flatten().tolist()\n", "\n", "\n", "def rescale_frame(frame, percent=50):\n", " '''\n", " Rescale a frame to a certain percentage compare to its original frame\n", " '''\n", " width = int(frame.shape[1] * percent/ 100)\n", " height = int(frame.shape[0] * percent/ 100)\n", " dim = (width, height)\n", " return cv2.resize(frame, dim, interpolation =cv2.INTER_AREA)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2. Analyze and detection bad pose\n", "\n", "Look through [this](./analyze_bad_pose.ipynb) on how we analyze bad foot and knee placement while doing squat." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import math \n", "\n", "\n", "def calculate_distance(pointX, pointY) -> float:\n", " '''\n", " Calculate a distance between 2 points\n", " '''\n", "\n", " x1, y1 = pointX\n", " x2, y2 = pointY\n", "\n", " return math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)\n", "\n", "\n", "def analyze_foot_knee_placement(results, stage: str, foot_shoulder_ratio_thresholds: list, knee_foot_ratio_thresholds: dict, visibility_threshold: int) -> dict:\n", " '''\n", " Calculate the ratio between the foot and shoulder for FOOT PLACEMENT analysis\n", " \n", " Calculate the ratio between the knee and foot for KNEE PLACEMENT analysis\n", "\n", " Return result explanation:\n", " -1: Unknown result due to poor visibility\n", " 0: Correct knee placement\n", " 1: Placement too tight\n", " 2: Placement too wide\n", " '''\n", " analyzed_results = {\n", " \"foot_placement\": -1,\n", " \"knee_placement\": -1,\n", " }\n", "\n", " landmarks = results.pose_landmarks.landmark\n", "\n", " # * Visibility check of important landmarks for foot placement analysis\n", " left_foot_index_vis = landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].visibility\n", " right_foot_index_vis = landmarks[mp_pose.PoseLandmark.RIGHT_FOOT_INDEX.value].visibility\n", "\n", " left_knee_vis = landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].visibility\n", " right_knee_vis = landmarks[mp_pose.PoseLandmark.RIGHT_KNEE.value].visibility\n", "\n", " # If visibility of any keypoints is low cancel the analysis\n", " if (left_foot_index_vis < visibility_threshold or right_foot_index_vis < visibility_threshold or left_knee_vis < visibility_threshold or right_knee_vis < visibility_threshold):\n", " return analyzed_results\n", " \n", " # * Calculate shoulder width\n", " left_shoulder = [landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x, landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y]\n", " right_shoulder = [landmarks[mp_pose.PoseLandmark.RIGHT_SHOULDER.value].x, landmarks[mp_pose.PoseLandmark.RIGHT_SHOULDER.value].y]\n", " shoulder_width = calculate_distance(left_shoulder, right_shoulder)\n", "\n", " # * Calculate 2-foot width\n", " left_foot_index = [landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].x, landmarks[mp_pose.PoseLandmark.LEFT_FOOT_INDEX.value].y]\n", " right_foot_index = [landmarks[mp_pose.PoseLandmark.RIGHT_FOOT_INDEX.value].x, landmarks[mp_pose.PoseLandmark.RIGHT_FOOT_INDEX.value].y]\n", " foot_width = calculate_distance(left_foot_index, right_foot_index)\n", "\n", " # * Calculate foot and shoulder ratio\n", " foot_shoulder_ratio = round(foot_width / shoulder_width, 1)\n", "\n", " # * Analyze FOOT PLACEMENT\n", " min_ratio_foot_shoulder, max_ratio_foot_shoulder = foot_shoulder_ratio_thresholds\n", " if min_ratio_foot_shoulder <= foot_shoulder_ratio <= max_ratio_foot_shoulder:\n", " analyzed_results[\"foot_placement\"] = 0\n", " elif foot_shoulder_ratio < min_ratio_foot_shoulder:\n", " analyzed_results[\"foot_placement\"] = 1\n", " elif foot_shoulder_ratio > max_ratio_foot_shoulder:\n", " analyzed_results[\"foot_placement\"] = 2\n", " \n", " # * Visibility check of important landmarks for knee placement analysis\n", " left_knee_vis = landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].visibility\n", " right_knee_vis = landmarks[mp_pose.PoseLandmark.RIGHT_KNEE.value].visibility\n", "\n", " # If visibility of any keypoints is low cancel the analysis\n", " if (left_knee_vis < visibility_threshold or right_knee_vis < visibility_threshold):\n", " print(\"Cannot see foot\")\n", " return analyzed_results\n", "\n", " # * Calculate 2 knee width\n", " left_knee = [landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].x, landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value].y]\n", " right_knee = [landmarks[mp_pose.PoseLandmark.RIGHT_KNEE.value].x, landmarks[mp_pose.PoseLandmark.RIGHT_KNEE.value].y]\n", " knee_width = calculate_distance(left_knee, right_knee)\n", "\n", " # * Calculate foot and shoulder ratio\n", " knee_foot_ratio = round(knee_width / foot_width, 1)\n", "\n", " # * Analyze KNEE placement\n", " up_min_ratio_knee_foot, up_max_ratio_knee_foot = knee_foot_ratio_thresholds.get(\"up\")\n", " middle_min_ratio_knee_foot, middle_max_ratio_knee_foot = knee_foot_ratio_thresholds.get(\"middle\")\n", " down_min_ratio_knee_foot, down_max_ratio_knee_foot = knee_foot_ratio_thresholds.get(\"down\")\n", "\n", " if stage == \"up\":\n", " if up_min_ratio_knee_foot <= knee_foot_ratio <= up_max_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 0\n", " elif knee_foot_ratio < up_min_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 1\n", " elif knee_foot_ratio > up_max_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 2\n", " elif stage == \"middle\":\n", " if middle_min_ratio_knee_foot <= knee_foot_ratio <= middle_max_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 0\n", " elif knee_foot_ratio < middle_min_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 1\n", " elif knee_foot_ratio > middle_max_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 2\n", " elif stage == \"down\":\n", " if down_min_ratio_knee_foot <= knee_foot_ratio <= down_max_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 0\n", " elif knee_foot_ratio < down_min_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 1\n", " elif knee_foot_ratio > down_max_ratio_knee_foot:\n", " analyzed_results[\"knee_placement\"] = 2\n", " \n", " return analyzed_results\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3. Make detection" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "VIDEO_PATH1 = \"../data/squat/squat_test_1.mov\"\n", "VIDEO_PATH2 = \"../data/squat/squat_test_2.mov\"\n", "VIDEO_PATH3 = \"../data/squat/squat_test_3.mp4\"\n", "VIDEO_PATH4 = \"../data/squat/squat_right_2.mp4\"" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# Load model for counter\n", "with open(\"./model/squat_model.pkl\", \"rb\") as f:\n", " count_model = pickle.load(f)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "cap = cv2.VideoCapture(VIDEO_PATH3)\n", "\n", "# Counter vars\n", "counter = 0\n", "current_stage = \"\"\n", "PREDICTION_PROB_THRESHOLD = 0.7\n", "\n", "# Error vars\n", "VISIBILITY_THRESHOLD = 0.6\n", "FOOT_SHOULDER_RATIO_THRESHOLDS = [1.2, 2.8]\n", "KNEE_FOOT_RATIO_THRESHOLDS = {\n", " \"up\": [0.5, 1.0],\n", " \"middle\": [0.7, 1.0],\n", " \"down\": [0.7, 1.1],\n", "}\n", "\n", "\n", "with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:\n", " while cap.isOpened():\n", " ret, image = cap.read()\n", "\n", " if not ret:\n", " break\n", " \n", " # Reduce size of a frame\n", " image = rescale_frame(image, 100)\n", "\n", " # Recolor image from BGR to RGB for mediapipe\n", " image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n", " image.flags.writeable = False\n", "\n", " results = pose.process(image)\n", " if not results.pose_landmarks:\n", " continue\n", "\n", " # Recolor image from BGR to RGB for mediapipe\n", " image.flags.writeable = True\n", " image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n", "\n", " # Draw landmarks and connections\n", " mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS, mp_drawing.DrawingSpec(color=(244, 117, 66), thickness=2, circle_radius=2), mp_drawing.DrawingSpec(color=(245, 66, 230), thickness=2, circle_radius=1))\n", "\n", " # Make detection\n", " try:\n", " # * Model prediction for SQUAT counter\n", " # Extract keypoints from frame for the input\n", " row = extract_important_keypoints(results)\n", " X = pd.DataFrame([row], columns=headers[1:])\n", "\n", " # Make prediction and its probability\n", " predicted_class = count_model.predict(X)[0]\n", " prediction_probabilities = count_model.predict_proba(X)[0]\n", " prediction_probability = round(prediction_probabilities[prediction_probabilities.argmax()], 2)\n", "\n", " # Evaluate model prediction\n", " if predicted_class == \"down\" and prediction_probability >= PREDICTION_PROB_THRESHOLD:\n", " current_stage = \"down\"\n", " elif current_stage == \"down\" and predicted_class == \"up\" and prediction_probability >= PREDICTION_PROB_THRESHOLD: \n", " current_stage = \"up\"\n", " counter += 1\n", "\n", " # Analyze squat pose\n", " analyzed_results = analyze_foot_knee_placement(results=results, stage=current_stage, foot_shoulder_ratio_thresholds=FOOT_SHOULDER_RATIO_THRESHOLDS, knee_foot_ratio_thresholds=KNEE_FOOT_RATIO_THRESHOLDS, visibility_threshold=VISIBILITY_THRESHOLD)\n", "\n", " foot_placement_evaluation = analyzed_results[\"foot_placement\"]\n", " knee_placement_evaluation = analyzed_results[\"knee_placement\"]\n", " \n", " # * Evaluate FOOT PLACEMENT error\n", " if foot_placement_evaluation == -1:\n", " foot_placement = \"UNK\"\n", " elif foot_placement_evaluation == 0:\n", " foot_placement = \"Correct\"\n", " elif foot_placement_evaluation == 1:\n", " foot_placement = \"Too tight\"\n", " elif foot_placement_evaluation == 2:\n", " foot_placement = \"Too wide\"\n", " \n", " # * Evaluate KNEE PLACEMENT error\n", " if knee_placement_evaluation == -1:\n", " knee_placement = \"UNK\"\n", " elif knee_placement_evaluation == 0:\n", " knee_placement = \"Correct\"\n", " elif knee_placement_evaluation == 1:\n", " knee_placement = \"Too tight\"\n", " elif knee_placement_evaluation == 2:\n", " knee_placement = \"Too wide\"\n", " \n", " # Visualization\n", " # Status box\n", " cv2.rectangle(image, (0, 0), (500, 60), (245, 117, 16), -1)\n", "\n", " # Display class\n", " cv2.putText(image, \"COUNT\", (10, 12), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)\n", " cv2.putText(image, f'{str(counter)}, {predicted_class.split(\" \")[0]}, {str(prediction_probability)}', (5, 40), cv2.FONT_HERSHEY_COMPLEX, .7, (255, 255, 255), 2, cv2.LINE_AA)\n", "\n", " # Display Foot and Shoulder width ratio\n", " cv2.putText(image, \"FOOT\", (200, 12), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)\n", " cv2.putText(image, foot_placement, (195, 40), cv2.FONT_HERSHEY_COMPLEX, .7, (255, 255, 255), 2, cv2.LINE_AA)\n", "\n", " # Display knee and Shoulder width ratio\n", " cv2.putText(image, \"KNEE\", (330, 12), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)\n", " cv2.putText(image, knee_placement, (325, 40), cv2.FONT_HERSHEY_COMPLEX, .7, (255, 255, 255), 2, cv2.LINE_AA)\n", "\n", " except Exception as e:\n", " print(f\"Error: {e}\")\n", " \n", " cv2.imshow(\"CV2\", image)\n", " \n", " # Press Q to close cv2 window\n", " if cv2.waitKey(1) & 0xFF == ord('q'):\n", " break\n", "\n", " cap.release()\n", " cv2.destroyAllWindows()\n", "\n", " # (Optional)Fix bugs cannot close windows in MacOS (https://stackoverflow.com/questions/6116564/destroywindow-does-not-close-window-on-mac-using-python-and-opencv)\n", " for i in range (1, 5):\n", " cv2.waitKey(1)\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.8.13 (conda)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "9260f401923fb5c4108c543a7d176de9733d378b3752e49535ad7c43c2271b65" } } }, "nbformat": 4, "nbformat_minor": 2 }