diff --git a/camera.py b/camera.py new file mode 100644 index 0000000..e69de29 diff --git a/example.py b/example.py index a0dd399..3e79a71 100644 --- a/example.py +++ b/example.py @@ -5,6 +5,7 @@ import math from model import * from renderer import * from dataset import * +from utils import get_smpl_joint ascii_logo = """\ /$$$$$$ /$$ /$$ /$$$$$$$ /$$ /$$ /$$ @@ -28,6 +29,91 @@ def load_config(): return config +def align_torso(): + print("TODO: align torso here, use below code as inpsiration") + + # # rotate to match body initial position orientation + # keypoints = keypoints * [1, -1, 0] + + # left_hip_2d = get_smpl_joint(keypoints, "hip-left") + # right_hip_2d = get_smpl_joint(keypoints, "hip-right") + + # betas = torch.randn([1, 10], dtype=torch.float32) + + # # body_pose = torch.randn((1, 69), dtype=torch.float32) * 0.7 + # body_pose = torch.zeros((1, 69)) + + # m.reset_params( + # betas=betas, + # body_pose=body_pose, + # ) + + # dtype = m.global_orient.dtype + + # global_orientation = m.global_orient.detach().cpu().numpy().squeeze() + # joints_transl = m.transl.detach().cpu().numpy().squeeze() + # print(global_orientation) + + # r.display_model(m, betas=betas, + # keypoints=keypoints, body_pose=body_pose) + + # model_out = m() + # joints = model_out.joints.detach().cpu().numpy().squeeze() + # left_knee = get_smpl_joint(joints, "chest") + # left_hip_joint = get_smpl_joint( + # joints, "hip-left") + # right_hip_joint = get_smpl_joint(joints, "hip-right") + + # print("Error:", np.linalg.norm(right_hip_2d - right_hip_joint) + # ** 2 + np.linalg.norm(left_hip_2d - left_hip_joint) ** 2) + + # transl = torch.tensor( + # [left_hip_joint - left_knee], dtype=m.transl.dtype) + + # # print("left-hip", left_hip_joint) + # # print("model trans:", m.transl) + # # print("trans:", [left_hip_2d - left_hip_joint]) + + # m.reset_params( + # transl=-transl + # ) + # model_out = m() + # # print(get_smpl_joint(joints, "hip-left")) + + # joints = model_out.joints.detach().cpu().numpy().squeeze() + # global_orientation = m.global_orient.detach().cpu().numpy().squeeze() + # left_hip_joint = get_smpl_joint(joints, "hip-left") + # right_hip_joint = get_smpl_joint(joints, "hip-right") + + # print("left-hip", left_hip_joint) + # print("model trans:", m.transl) + + # print("Error Left:", np.linalg.norm(left_hip_2d - left_hip_joint) ** 2) + + # print("Error:", np.linalg.norm(right_hip_2d - right_hip_joint) + # ** 2 + np.linalg.norm(left_hip_2d - left_hip_joint) ** 2) + + # # try moving left elbow + # # body_pose.reshape(23, 3)[17][0] = math.pi / 4 + # # body_pose.reshape(23, 3)[17][1] = - math.pi / 2 + # # body_pose.reshape(23, 3)[17][2] = math.pi / 4 + + # # # right knee + # # body_pose.reshape(23, 3)[3][0] = math.pi / 4 + # # body_pose.reshape(23, 3)[3][1] = - math.pi / 2 + # # body_pose.reshape(23, 3)[3][2] = math.pi / 4 + + # # # left knee + # # body_pose.reshape(23, 3)[3][0] = math.pi / 4 + # # body_pose.reshape(23, 3)[3][1] = - math.pi / 2 + # # body_pose.reshape(23, 3)[3][2] = math.pi / 4 + + # # body_pose = torch.from_numpy(keypoints.reshape((1, 69))) + # # print(body_pose) + # r.display_model(m, betas=betas, + # keypoints=keypoints, body_pose=body_pose) + + def main(): print(ascii_logo) conf = load_config() @@ -37,33 +123,12 @@ def main(): l = SMPLyModel(conf['modelPath']) r = SMPLyRenderer() - m = l.create_model() - + model = l.create_model() keypoints, conf = dataset[0] - print(keypoints, conf) - betas = torch.randn([1, 10], dtype=torch.float32) - # body_pose = torch.randn((1, 69), dtype=torch.float32) * 0.7 - body_pose = torch.zeros((1, 69)) + model_out = model() - # try moving left elbow - # body_pose.reshape(23, 3)[17][0] = math.pi / 4 - # body_pose.reshape(23, 3)[17][1] = - math.pi / 2 - # body_pose.reshape(23, 3)[17][2] = math.pi / 4 - - # # right knee - # body_pose.reshape(23, 3)[3][0] = math.pi / 4 - # body_pose.reshape(23, 3)[3][1] = - math.pi / 2 - # body_pose.reshape(23, 3)[3][2] = math.pi / 4 - - # # left knee - # body_pose.reshape(23, 3)[3][0] = math.pi / 4 - # body_pose.reshape(23, 3)[3][1] = - math.pi / 2 - # body_pose.reshape(23, 3)[3][2] = math.pi / 4 - - # body_pose = torch.from_numpy(keypoints.reshape((1, 69))) - print(body_pose) - r.display_model(m, betas=betas, keypoints=keypoints, body_pose=body_pose) + # print(keypoints, conf) if __name__ == '__main__': diff --git a/example_fitter.py b/example_fitter.py new file mode 100644 index 0000000..94fb992 --- /dev/null +++ b/example_fitter.py @@ -0,0 +1,120 @@ + + +import yaml +import torch +import math + +from model import * +# from renderer import * +from dataset import * +from utils import get_named_joint + +ascii_logo = """\ + /$$$$$$ /$$ /$$ /$$$$$$$ /$$ /$$ /$$ + /$$__ $$| $$$ /$$$| $$__ $$| $$ | $$ /$$/ +| $$ \__/| $$$$ /$$$$| $$ \ $$| $$ \ $$ /$$/ +| $$$$$$ | $$ $$/$$ $$| $$$$$$$/| $$ \ $$$$/ + \____ $$| $$ $$$| $$| $$____/ | $$ \ $$/ + /$$ \ $$| $$\ $ | $$| $$ | $$ | $$ +| $$$$$$/| $$ \/ | $$| $$ | $$$$$$$$| $$ + \______/ |__/ |__/|__/ |________/|__/ + +""" + + +def renderPoints(scene, points, radius=0.005, colors=[0.0, 0.0, 1.0, 1.0], name=None): + sm = trimesh.creation.uv_sphere(radius=radius) + sm.visual.vertex_colors = colors + tfs = np.tile(np.eye(4), (len(points), 1, 1)) + tfs[:, :3, 3] = points + pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs) + # return the render scene node + return scene.add(pcl, name=name) + + +def load_config(): + with open('./config.yaml') as file: + # The FullLoader parameter handles the conversion from YAML + # scalar values to Python the dictionary format + config = yaml.load(file, Loader=yaml.FullLoader) + + return config + + +print(ascii_logo) +conf = load_config() +print("config loaded") +dataset = SMPLyDataset() + +# FIND OPENPOSE TO SMPL MAPPINGS +mapping = [24, 12, 17, 19, 21, 16, 18, 20, 0, 2, 5, 8, 1, 4, + 7, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34] + +arr = np.ones(45) * -1 + +for i, v in enumerate(mapping): + arr[v] = i + print(v, i) + +for v in arr: + print( + int(v), "," + ) +print(arr) + +# ------------------------------ +# Load data +# ------------------------------ +l = SMPLyModel(conf['modelPath']) +model = l.create_model() +keypoints, conf = dataset[0] +print("keypoints shape:", keypoints.shape) +# --------------------------------- +# Generate model and get joints +# --------------------------------- +model_out = model() +joints = model_out.joints.detach().cpu().numpy().squeeze() + +# --------------------------------- +# Draw in the joints of interest +# --------------------------------- + +# SMPL joint positions (cl = chest left, cr = chest right) +cl_joint = get_named_joint(joints, "elbow-left") +cr_joint = get_named_joint(joints, "hip-left") + +# keypoint joint position +cl_point = get_named_joint(keypoints, "shoulder-left", type="body_25") +cr_point = get_named_joint(keypoints, "shoulder-right", type="body_25") + +# create joints copy without points of interest +other_joints = np.array([joints[x] + for x in range(len(joints)) if x < 13 or x > 14]) +print("removed other joints from list:", len( + other_joints), other_joints.shape, len(joints)) +scene = pyrender.Scene() + +renderPoints(scene, other_joints) + +renderPoints(scene, [cl_joint, cr_joint], + radius=0.01, colors=[1.0, 0.0, 1.0, 1.0]) + +v = pyrender.Viewer(scene, + use_raymond_lighting=True, + # show_world_axis=True + run_in_thread=False + ) + + +# ------------------------------------- +# Optimize for translation and rotation +# ------------------------------------- + +# ----------------------------- +# Render the points +# ----------------------------- +v = pyrender.Viewer(scene, + use_raymond_lighting=True, + # show_world_axis=True + run_in_thread=True + ) diff --git a/model.py b/model.py index f354bdf..f1b0205 100644 --- a/model.py +++ b/model.py @@ -48,6 +48,8 @@ class SMPLyModel(): use_face_contour=self.use_face_contour, create_body_pose=True, num_betas=self.num_betas, + return_verts=True, + # global_orient=torch.randn(3), num_expression_coeffs=self.num_expression_coeffs, ext=self.ext) print(self.model) diff --git a/optimizer.py b/optimizer.py new file mode 100644 index 0000000..b9e1769 --- /dev/null +++ b/optimizer.py @@ -0,0 +1,14 @@ +import torch +import torch.nn as nn + + +class CameraLoss(nn.Module): + def __init__(self): + print("CameraLoss init") + + def forward(self, keypoints, joints, transform): + + # simple stupid implementation + # compute 2d distance between model and openpose joints + # ignoring Z axis + joints * transform diff --git a/preview.png b/preview.png new file mode 100644 index 0000000..23ae55d Binary files /dev/null and b/preview.png differ diff --git a/renderer.py b/renderer.py index f554164..5d20115 100644 --- a/renderer.py +++ b/renderer.py @@ -15,17 +15,11 @@ class SMPLyRenderer(): # TODO: use __call__ for this def render_model( self, - model: smplx.SMPL, + model, betas: torch.TensorType, body_pose: torch.TensorType, ): - # compute model - model_out = model( - betas=betas, - body_pose=body_pose, - return_verts=True, - ) - + model_out = model() # TODO: check if this also works with CUDA vertices = model_out.vertices.detach().cpu().numpy().squeeze() joints = model_out.joints.detach().cpu().numpy().squeeze() @@ -39,6 +33,38 @@ class SMPLyRenderer(): return (tri_mesh, joints, vertices) + def set_keypoints(self, keypoints): + scene = self.scene + self.viewer.render_lock.acquire() + + if self.keypoints_node is not None: + scene.remove(self.keypoints_node) + + sm = trimesh.creation.uv_sphere(radius=0.01) + sm.visual.vertex_colors = [0.0, 0.0, 1.0, 1.0] + tfs = np.tile(np.eye(4), (len(keypoints), 1, 1)) + tfs[:, :3, 3] = keypoints + keypoints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs) + self.keypoints_node = scene.add(keypoints_pcl, name="keypoints") + + self.viewer.render_lock.release() + + def set_model(self, model): + scene = self.scene + self.viewer.render_lock.acquire() + + if self.keypoints_node is not None: + scene.remove(self.keypoints_node) + + sm = trimesh.creation.uv_sphere(radius=0.01) + sm.visual.vertex_colors = [0.0, 0.0, 1.0, 1.0] + tfs = np.tile(np.eye(4), (len(keypoints), 1, 1)) + tfs[:, :3, 3] = keypoints + keypoints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs) + self.keypoints_node = scene.add(keypoints_pcl, name="keypoints") + + self.viewer.render_lock.release() + def display_mesh( self, tri_mesh: trimesh.Trimesh, @@ -46,40 +72,56 @@ class SMPLyRenderer(): keypoints=None, render_openpose_wireframe=True, ): - mesh = pyrender.Mesh.from_trimesh(tri_mesh) scene = pyrender.Scene() - scene.add(mesh) - + self.body_node = scene.add(mesh, name="body_mesh") if joints is not None: sm = trimesh.creation.uv_sphere(radius=0.005) sm.visual.vertex_colors = [0.9, 0.1, 0.1, 1.0] tfs = np.tile(np.eye(4), (len(joints), 1, 1)) tfs[:, :3, 3] = joints joints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs) - scene.add(joints_pcl) + self.joints_node = scene.add(joints_pcl, name="joints") if keypoints is not None: - sm = trimesh.creation.uv_sphere(radius=0.01) - sm.visual.vertex_colors = [0.0, 0.0, 1.0, 1.0] - tfs = np.tile(np.eye(4), (len(keypoints), 1, 1)) - tfs[:, :3, 3] = keypoints - keypoints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs) - scene.add(keypoints_pcl) - pyrender.Viewer(scene, - use_raymond_lighting=True, - # show_world_axis=True - ) + self.start_render(scene) + + def start_render( + self, + scene, + ): + self.scene = scene + self.viewer = pyrender.Viewer(scene, + use_raymond_lighting=True, + # show_world_axis=True + run_in_thread=True + ) + # self.update() + while True: + pass + + def update(self): + pose = self.body_node.get_pose() + print(pose) + self.viewer.render_lock.acquire() + self.scene.set_pose(self.body_node, pose) + self.viewer.render_lock.release() + + def end_scene(self): + self.viewer.close_external() + while self.viewer.is_active: + pass def display_model( self, - model: smplx.SMPL, + model, betas: torch.TensorType, body_pose: torch.TensorType, keypoints=None, ): - (tri_mesh, joints, vertices) = self.render_model(model, betas, body_pose) + (tri_mesh, joints, vertices) = self.render_model( + model, betas, body_pose) self.display_mesh(tri_mesh, joints, keypoints) diff --git a/utils.py b/utils.py index f0f04a5..d5809fe 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,54 @@ from typing import List, Set, Dict, Tuple, Optional import numpy as np +openpose_to_smpl = np.array([ + 8, # hip - middle + 12, # hip - right + 9, # hip - left + -1, # body center (belly, not present in body_25) + 13, # left knee + 10, # right knee, + -1, + 14, # left ankle + 11, # right ankle + -1, + -1, + -1, + 1, # chest + -1, + -1, + -1, + 5, # left shoulder + 2, # right shoulder + 6, # left elbow + 3, # right elbow + 7, # left hand + 4, # right hand + -1, + -1, + 0, # head + 15, + 16, + 17, + 18, + 19, # left toe + 20, + 21, + 22, # right toe + 23, + 24, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, +]) + def get_mapping_arr( input_format: str = "body_25", @@ -9,32 +57,46 @@ def get_mapping_arr( # TODO: expand features as needed # based on mappings found here # https://github.com/ortegatron/playing_smplifyx/blob/master/smplifyx/utils.py - return np.array([ - # 8, # hip - middle - 9, # hip - left - 12, # hip - right - -1, # body center (belly, not present in body_25) - 13, # left knee - 10, # right knee, - 1, # chest - 14, # left ankle - 11, # right ankle - 1, # chest again ? check this one out - 19, # left toe - 22, # right toe - -1, # neck (not present in body_25) - -1, # between torso and left shoulder - -1, # between torso and right shoulder - 0, # head - 5, # left shoulder - 2, # right shoulder - 6, # left elbow - 3, # right elbow - 7, # left hand - 4, # right hand - -1, # left fingers - -1 # right fingers - ]) + return openpose_to_smpl + + +joint_names_body_25 = { + "hip-left": 9, + "hip-right": 12, + "belly": 8, + "knee-left": 10, + "knee-right": 13, + "ankle-left": 11, + "ankle-right": 14, + "toes-left": 22, + "toes-right": 19, + "neck": 1, + "head": 0, + "shoulder-left": 2, + "shoulder-right": 5, + "elbow-left": 3, + "elbow-right": 6, + "hand-left": 4, + "hand-right": 7, +} + + +def get_named_joint(joints: List, name: str, type="smpl"): + """get SMPL joint by name + + Args: + joints (List): list of SMPL joints + name (str): joint to be extracted + + Returns: + Tuple[float, float, float]: Coordinates of the selected joint + """ + if type == "smpl": + mapping = get_mapping_arr() + index = joint_names_body_25[name] + return joints[np.where(mapping == index)] + if type == "body_25": + return joints[joint_names_body_25[name]] def apply_mapping( @@ -62,8 +124,8 @@ def openpose_to_opengl_coords( points = np.array([ [ - x / real_width * 2 - 1, - y / real_height * 2, + x / real_width, + y / real_height, 0 ] for (x, y, z) in input_data])