vhad/body-pose-animation
1
0
Fork 0
mirror of https://github.com/gosticks/body-pose-animation.git synced 2025-10-16 11:45:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

WIP: fixing mapping for camera solver

Browse Source
This commit is contained in:
Wlad Meixner 2021-01-15 22:47:15 +01:00 committed by Wlad
parent 324f167d69
commit a46babe38b
8 changed files with 381 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
camera.py Normal file
View File

113
example.py
View File

@ -5,6 +5,7 @@ import math
from model import *
from renderer import *
from dataset import *
from utils import get_smpl_joint
ascii_logo = """\
/$$$$$$ /$$ /$$ /$$$$$$$ /$$ /$$ /$$
@ -28,6 +29,91 @@ def load_config():
return config
def align_torso():
print("TODO: align torso here, use below code as inpsiration")
# # rotate to match body initial position orientation
# keypoints = keypoints * [1, -1, 0]
# left_hip_2d = get_smpl_joint(keypoints, "hip-left")
# right_hip_2d = get_smpl_joint(keypoints, "hip-right")
# betas = torch.randn([1, 10], dtype=torch.float32)
# # body_pose = torch.randn((1, 69), dtype=torch.float32) * 0.7
# body_pose = torch.zeros((1, 69))
# m.reset_params(
# betas=betas,
# body_pose=body_pose,
# )
# dtype = m.global_orient.dtype
# global_orientation = m.global_orient.detach().cpu().numpy().squeeze()
# joints_transl = m.transl.detach().cpu().numpy().squeeze()
# print(global_orientation)
# r.display_model(m, betas=betas,
# keypoints=keypoints, body_pose=body_pose)
# model_out = m()
# joints = model_out.joints.detach().cpu().numpy().squeeze()
# left_knee = get_smpl_joint(joints, "chest")
# left_hip_joint = get_smpl_joint(
# joints, "hip-left")
# right_hip_joint = get_smpl_joint(joints, "hip-right")
# print("Error:", np.linalg.norm(right_hip_2d - right_hip_joint)
# ** 2 + np.linalg.norm(left_hip_2d - left_hip_joint) ** 2)
# transl = torch.tensor(
# [left_hip_joint - left_knee], dtype=m.transl.dtype)
# # print("left-hip", left_hip_joint)
# # print("model trans:", m.transl)
# # print("trans:", [left_hip_2d - left_hip_joint])
# m.reset_params(
# transl=-transl
# )
# model_out = m()
# # print(get_smpl_joint(joints, "hip-left"))
# joints = model_out.joints.detach().cpu().numpy().squeeze()
# global_orientation = m.global_orient.detach().cpu().numpy().squeeze()
# left_hip_joint = get_smpl_joint(joints, "hip-left")
# right_hip_joint = get_smpl_joint(joints, "hip-right")
# print("left-hip", left_hip_joint)
# print("model trans:", m.transl)
# print("Error Left:", np.linalg.norm(left_hip_2d - left_hip_joint) ** 2)
# print("Error:", np.linalg.norm(right_hip_2d - right_hip_joint)
# ** 2 + np.linalg.norm(left_hip_2d - left_hip_joint) ** 2)
# # try moving left elbow
# # body_pose.reshape(23, 3)[17][0] = math.pi / 4
# # body_pose.reshape(23, 3)[17][1] = - math.pi / 2
# # body_pose.reshape(23, 3)[17][2] = math.pi / 4
# # # right knee
# # body_pose.reshape(23, 3)[3][0] = math.pi / 4
# # body_pose.reshape(23, 3)[3][1] = - math.pi / 2
# # body_pose.reshape(23, 3)[3][2] = math.pi / 4
# # # left knee
# # body_pose.reshape(23, 3)[3][0] = math.pi / 4
# # body_pose.reshape(23, 3)[3][1] = - math.pi / 2
# # body_pose.reshape(23, 3)[3][2] = math.pi / 4
# # body_pose = torch.from_numpy(keypoints.reshape((1, 69)))
# # print(body_pose)
# r.display_model(m, betas=betas,
# keypoints=keypoints, body_pose=body_pose)
def main():
print(ascii_logo)
conf = load_config()
@ -37,33 +123,12 @@ def main():
l = SMPLyModel(conf['modelPath'])
r = SMPLyRenderer()
m = l.create_model()
model = l.create_model()
keypoints, conf = dataset[0]
print(keypoints, conf)
betas = torch.randn([1, 10], dtype=torch.float32)
# body_pose = torch.randn((1, 69), dtype=torch.float32) * 0.7
body_pose = torch.zeros((1, 69))
model_out = model()
# try moving left elbow
# body_pose.reshape(23, 3)[17][0] = math.pi / 4
# body_pose.reshape(23, 3)[17][1] = - math.pi / 2
# body_pose.reshape(23, 3)[17][2] = math.pi / 4
# # right knee
# body_pose.reshape(23, 3)[3][0] = math.pi / 4
# body_pose.reshape(23, 3)[3][1] = - math.pi / 2
# body_pose.reshape(23, 3)[3][2] = math.pi / 4
# # left knee
# body_pose.reshape(23, 3)[3][0] = math.pi / 4
# body_pose.reshape(23, 3)[3][1] = - math.pi / 2
# body_pose.reshape(23, 3)[3][2] = math.pi / 4
# body_pose = torch.from_numpy(keypoints.reshape((1, 69)))
print(body_pose)
r.display_model(m, betas=betas, keypoints=keypoints, body_pose=body_pose)
# print(keypoints, conf)
if __name__ == '__main__':

120
example_fitter.py Normal file
View File

@ -0,0 +1,120 @@
import yaml
import torch
import math
from model import *
# from renderer import *
from dataset import *
from utils import get_named_joint
ascii_logo = """\
/$$$$$$ /$$ /$$ /$$$$$$$ /$$ /$$ /$$
/$$__ $$| $$$ /$$$| $$__ $$| $$ | $$ /$$/
| $$ \__/| $$$$ /$$$$| $$ \ $$| $$ \ $$ /$$/
| $$$$$$ | $$ $$/$$ $$| $$$$$$$/| $$ \ $$$$/
\____ $$| $$ $$$| $$| $$____/ | $$ \ $$/
/$$ \ $$| $$\ $ | $$| $$ | $$ | $$
| $$$$$$/| $$ \/ | $$| $$ | $$$$$$$$| $$
\______/ |__/ |__/|__/ |________/|__/
"""
def renderPoints(scene, points, radius=0.005, colors=[0.0, 0.0, 1.0, 1.0], name=None):
sm = trimesh.creation.uv_sphere(radius=radius)
sm.visual.vertex_colors = colors
tfs = np.tile(np.eye(4), (len(points), 1, 1))
tfs[:, :3, 3] = points
pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
# return the render scene node
return scene.add(pcl, name=name)
def load_config():
with open('./config.yaml') as file:
# The FullLoader parameter handles the conversion from YAML
# scalar values to Python the dictionary format
config = yaml.load(file, Loader=yaml.FullLoader)
return config
print(ascii_logo)
conf = load_config()
print("config loaded")
dataset = SMPLyDataset()
# FIND OPENPOSE TO SMPL MAPPINGS
mapping = [24, 12, 17, 19, 21, 16, 18, 20, 0, 2, 5, 8, 1, 4,
7, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
arr = np.ones(45) * -1
for i, v in enumerate(mapping):
arr[v] = i
print(v, i)
for v in arr:
print(
int(v), ","
)
print(arr)
# ------------------------------
# Load data
# ------------------------------
l = SMPLyModel(conf['modelPath'])
model = l.create_model()
keypoints, conf = dataset[0]
print("keypoints shape:", keypoints.shape)
# ---------------------------------
# Generate model and get joints
# ---------------------------------
model_out = model()
joints = model_out.joints.detach().cpu().numpy().squeeze()
# ---------------------------------
# Draw in the joints of interest
# ---------------------------------
# SMPL joint positions (cl = chest left, cr = chest right)
cl_joint = get_named_joint(joints, "elbow-left")
cr_joint = get_named_joint(joints, "hip-left")
# keypoint joint position
cl_point = get_named_joint(keypoints, "shoulder-left", type="body_25")
cr_point = get_named_joint(keypoints, "shoulder-right", type="body_25")
# create joints copy without points of interest
other_joints = np.array([joints[x]
for x in range(len(joints)) if x < 13 or x > 14])
print("removed other joints from list:", len(
other_joints), other_joints.shape, len(joints))
scene = pyrender.Scene()
renderPoints(scene, other_joints)
renderPoints(scene, [cl_joint, cr_joint],
radius=0.01, colors=[1.0, 0.0, 1.0, 1.0])
v = pyrender.Viewer(scene,
use_raymond_lighting=True,
# show_world_axis=True
run_in_thread=False
)
# -------------------------------------
# Optimize for translation and rotation
# -------------------------------------
# -----------------------------
# Render the points
# -----------------------------
v = pyrender.Viewer(scene,
use_raymond_lighting=True,
# show_world_axis=True
run_in_thread=True
)

2
model.py
View File

@ -48,6 +48,8 @@ class SMPLyModel():
use_face_contour=self.use_face_contour,
create_body_pose=True,
num_betas=self.num_betas,
return_verts=True,
# global_orient=torch.randn(3),
num_expression_coeffs=self.num_expression_coeffs,
ext=self.ext)
print(self.model)

14
optimizer.py Normal file
View File

@ -0,0 +1,14 @@
import torch
import torch.nn as nn
class CameraLoss(nn.Module):
def __init__(self):
print("CameraLoss init")
def forward(self, keypoints, joints, transform):
# simple stupid implementation
# compute 2d distance between model and openpose joints
# ignoring Z axis
joints * transform

BIN
preview.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 304 KiB

90
renderer.py
View File

@ -15,17 +15,11 @@ class SMPLyRenderer():
# TODO: use __call__ for this
def render_model(
self,
model: smplx.SMPL,
model,
betas: torch.TensorType,
body_pose: torch.TensorType,
):
# compute model
model_out = model(
betas=betas,
body_pose=body_pose,
return_verts=True,
)
model_out = model()
# TODO: check if this also works with CUDA
vertices = model_out.vertices.detach().cpu().numpy().squeeze()
joints = model_out.joints.detach().cpu().numpy().squeeze()
@ -39,6 +33,38 @@ class SMPLyRenderer():
return (tri_mesh, joints, vertices)
def set_keypoints(self, keypoints):
scene = self.scene
self.viewer.render_lock.acquire()
if self.keypoints_node is not None:
scene.remove(self.keypoints_node)
sm = trimesh.creation.uv_sphere(radius=0.01)
sm.visual.vertex_colors = [0.0, 0.0, 1.0, 1.0]
tfs = np.tile(np.eye(4), (len(keypoints), 1, 1))
tfs[:, :3, 3] = keypoints
keypoints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
self.keypoints_node = scene.add(keypoints_pcl, name="keypoints")
self.viewer.render_lock.release()
def set_model(self, model):
scene = self.scene
self.viewer.render_lock.acquire()
if self.keypoints_node is not None:
scene.remove(self.keypoints_node)
sm = trimesh.creation.uv_sphere(radius=0.01)
sm.visual.vertex_colors = [0.0, 0.0, 1.0, 1.0]
tfs = np.tile(np.eye(4), (len(keypoints), 1, 1))
tfs[:, :3, 3] = keypoints
keypoints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
self.keypoints_node = scene.add(keypoints_pcl, name="keypoints")
self.viewer.render_lock.release()
def display_mesh(
self,
tri_mesh: trimesh.Trimesh,
@ -46,40 +72,56 @@ class SMPLyRenderer():
keypoints=None,
render_openpose_wireframe=True,
):
mesh = pyrender.Mesh.from_trimesh(tri_mesh)
scene = pyrender.Scene()
scene.add(mesh)
self.body_node = scene.add(mesh, name="body_mesh")
if joints is not None:
sm = trimesh.creation.uv_sphere(radius=0.005)
sm.visual.vertex_colors = [0.9, 0.1, 0.1, 1.0]
tfs = np.tile(np.eye(4), (len(joints), 1, 1))
tfs[:, :3, 3] = joints
joints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
scene.add(joints_pcl)
self.joints_node = scene.add(joints_pcl, name="joints")
if keypoints is not None:
sm = trimesh.creation.uv_sphere(radius=0.01)
sm.visual.vertex_colors = [0.0, 0.0, 1.0, 1.0]
tfs = np.tile(np.eye(4), (len(keypoints), 1, 1))
tfs[:, :3, 3] = keypoints
keypoints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
scene.add(keypoints_pcl)
pyrender.Viewer(scene,
use_raymond_lighting=True,
# show_world_axis=True
)
self.start_render(scene)
def start_render(
self,
scene,
):
self.scene = scene
self.viewer = pyrender.Viewer(scene,
use_raymond_lighting=True,
# show_world_axis=True
run_in_thread=True
)
# self.update()
while True:
pass
def update(self):
pose = self.body_node.get_pose()
print(pose)
self.viewer.render_lock.acquire()
self.scene.set_pose(self.body_node, pose)
self.viewer.render_lock.release()
def end_scene(self):
self.viewer.close_external()
while self.viewer.is_active:
pass
def display_model(
self,
model: smplx.SMPL,
model,
betas: torch.TensorType,
body_pose: torch.TensorType,
keypoints=None,
):
(tri_mesh, joints, vertices) = self.render_model(model, betas, body_pose)
(tri_mesh, joints, vertices) = self.render_model(
model, betas, body_pose)
self.display_mesh(tri_mesh, joints, keypoints)

118
utils.py
View File

@ -1,6 +1,54 @@
from typing import List, Set, Dict, Tuple, Optional
import numpy as np
openpose_to_smpl = np.array([
8, # hip - middle
12, # hip - right
9, # hip - left
-1, # body center (belly, not present in body_25)
13, # left knee
10, # right knee,
-1,
14, # left ankle
11, # right ankle
-1,
-1,
-1,
1, # chest
-1,
-1,
-1,
5, # left shoulder
2, # right shoulder
6, # left elbow
3, # right elbow
7, # left hand
4, # right hand
-1,
-1,
0, # head
15,
16,
17,
18,
19, # left toe
20,
21,
22, # right toe
23,
24,
-1,
-1,
-1,
-1,
-1,
-1,
-1,
-1,
-1,
-1,
])
def get_mapping_arr(
input_format: str = "body_25",
@ -9,32 +57,46 @@ def get_mapping_arr(
# TODO: expand features as needed
# based on mappings found here
# https://github.com/ortegatron/playing_smplifyx/blob/master/smplifyx/utils.py
return np.array([
# 8, # hip - middle
9, # hip - left
12, # hip - right
-1, # body center (belly, not present in body_25)
13, # left knee
10, # right knee,
1, # chest
14, # left ankle
11, # right ankle
1, # chest again ? check this one out
19, # left toe
22, # right toe
-1, # neck (not present in body_25)
-1, # between torso and left shoulder
-1, # between torso and right shoulder
0, # head
5, # left shoulder
2, # right shoulder
6, # left elbow
3, # right elbow
7, # left hand
4, # right hand
-1, # left fingers
-1 # right fingers
])
return openpose_to_smpl
joint_names_body_25 = {
"hip-left": 9,
"hip-right": 12,
"belly": 8,
"knee-left": 10,
"knee-right": 13,
"ankle-left": 11,
"ankle-right": 14,
"toes-left": 22,
"toes-right": 19,
"neck": 1,
"head": 0,
"shoulder-left": 2,
"shoulder-right": 5,
"elbow-left": 3,
"elbow-right": 6,
"hand-left": 4,
"hand-right": 7,
}
def get_named_joint(joints: List, name: str, type="smpl"):
"""get SMPL joint by name
Args:
joints (List): list of SMPL joints
name (str): joint to be extracted
Returns:
Tuple[float, float, float]: Coordinates of the selected joint
"""
if type == "smpl":
mapping = get_mapping_arr()
index = joint_names_body_25[name]
return joints[np.where(mapping == index)]
if type == "body_25":
return joints[joint_names_body_25[name]]
def apply_mapping(
@ -62,8 +124,8 @@ def openpose_to_opengl_coords(
points = np.array([
[
x / real_width * 2 - 1,
y / real_height * 2,
x / real_width,
y / real_height,
0
] for (x, y, z) in input_data])