vhad/body-pose-animation
1
0
Fork 0
mirror of https://github.com/gosticks/body-pose-animation.git synced 2025-10-16 11:45:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

final preparations

Browse Source
This commit is contained in:
Wlad 2021-02-23 21:40:55 +01:00
parent 51f220b2ae
commit 316bc2d7e3
11 changed files with 232 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
config.yaml
View File

@ -1,5 +1,5 @@
output:
rootDir: "./tests/20-02-21"
rootDir: "./presentation"
prefix: ""
smpl:
modelRootDir: ./models
@ -9,25 +9,25 @@ smpl:
useVposerInit: false
data:
renameFiles: false
rootDir: ./samples/video02
rootDir: ./samples/video01
personId: 0
sampleImageFormat: "frame-%%%.png"
sampleNameFormat: "input2_%%%%%%%%%%%%_keypoints.json"
sampleImageFormat: "input_%%%%%%%%%%%%_rendered.png"
sampleNameFormat: "input_%%%%%%%%%%%%_keypoints.json"
sampleCoords: !!python/tuple [1080, 1080]
camera:
lr: 0.001
patience: 10
optimizer: Adam
orientation:
lr: 0.01
optimizer: Adam
iterations: 100
lr: 0.03
optimizer: LBFGS
iterations: 5
joint_names: ["hip-left", "hip-right", "shoulder-left", "shoulder-right"] # joints to be used for optimization
pose:
device: cuda
lr: 0.01
optimizer: Adam # currently supported Adam, LBFGS
iterations: 100
iterations: 150
useCameraIntrinsics: true
useOpenPoseConf: true # use openpose confidence to weight L2 distance loss
bodyMeanLoss:
@ -35,21 +35,21 @@ pose:
weight: 0.1
bodyPrior:
enabled: true
weight: 0.01
weight: 0.1
anglePrior:
enabled: true
weight: 0.2
weight: 0.05
# optional per joint configurations
angleIdx: [56, 53, 12, 9, 37, 40]
directions: [-1, 1, -1, -1, 1, -1]
directions: [1, -1, -1, -1, 1, -1]
# weights per joint
weights: [0.4, 0.4, 0.4, 0.4, 0.2, 0.2]
weights: [0.2, 0.2, 0.8, 0.8, 0.0, 0.0]
angleLimitLoss:
enabled: true
weight: 0.01
angleSumLoss:
enabled: true
weight: 0.01
weight: 0.001
intersectLoss:
enabled: true
weight: 0.5
@ -57,14 +57,14 @@ pose:
sigma: 0.5
changeLoss:
enabled: true
weight: 0.1
weight: 0.07
confWeights:
enabled: false
vposerPath: "./vposer_v1_0"
temporal:
enabled: true
iterations: 75
lr: 0.01
iterations: 50
lr: 0.03
preview:
enable: true,
keypoins:

81
config_loss_off.yaml Normal file
View File

@ -0,0 +1,81 @@
output:
rootDir: "./tests/final-03"
prefix: ""
smpl:
modelRootDir: ./models
type: smplx # possible options smplx and smpl
gender: MALE # possible options MALE, FEMALE, NEUTRAL
ext: npz
useVposerInit: false
data:
renameFiles: false
rootDir: ./samples/video01
personId: 0
sampleImageFormat: "input_%%%%%%%%%%%%_rendered.png"
sampleNameFormat: "input_%%%%%%%%%%%%_keypoints.json"
sampleCoords: !!python/tuple [1080, 1080]
camera:
lr: 0.001
patience: 10
optimizer: Adam
orientation:
lr: 0.03
optimizer: LBFGS
iterations: 5
joint_names: ["hip-left", "hip-right", "shoulder-left", "shoulder-right"] # joints to be used for optimization
pose:
device: cuda
lr: 0.01
optimizer: Adam # currently supported Adam, LBFGS
iterations: 200
useCameraIntrinsics: true
useOpenPoseConf: false # use openpose confidence to weight L2 distance loss
bodyMeanLoss:
enabled: false
weight: 0.1
bodyPrior:
enabled: false
weight: 0.01
anglePrior:
enabled: false
weight: 0.2
# optional per joint configurations
angleIdx: [56, 53, 12, 9, 37, 40]
directions: [-1, 1, -1, -1, 1, -1]
# weights per joint
weights: [0.4, 0.4, 0.4, 0.4, 0.2, 0.2]
angleLimitLoss:
enabled: false
weight: 0.01
angleSumLoss:
enabled: false
weight: 0.001
intersectLoss:
enabled: false
weight: 0.5
maxCollisions: 8
sigma: 0.5
changeLoss:
enabled: true
weight: 0.2
confWeights:
enabled: false
vposerPath: "./vposer_v1_0"
temporal:
enabled: true
iterations: 30
lr: 0.01
preview:
enable: true,
keypoins:
enable: true,
radius: 0.01
color: 1.0, 0.0, 1.0, 1.0
keypoint_torso:
enable: true,
radius: 0.01
color: 1.0, 0.0, 1.0, 1.0
joints:
enable: true
radius: 0.01
color: 0.0, 0.7, 0.0, 1.0

14
example_fit.py
View File

@ -1,4 +1,5 @@
import os
from utils.video import make_video, make_video_with_pip
from utils.graphs import render_loss_graph
from train import optimize_sample
@ -9,19 +10,28 @@ from dataset import SMPLyDataset
# load and select sample
config = load_config()
dataset = SMPLyDataset.from_config(config=config)
sample_index = 55
sample_index = 150
save_to_video = True
if os.getenv('SAMPLE_INDEX') is not None:
sample_index = int(os.getenv('SAMPLE_INDEX'))
# train for pose
pose, camera_transformation, loss_history, step_imgs, loss_components = optimize_sample(
sample_index,
dataset,
config,
interactive=True
interactive=not save_to_video,
offscreen=save_to_video
)
if save_to_video:
img_path = dataset.get_image_path(sample_index)
make_video_with_pip(step_imgs, pip_image_path=img_path,
video_name="example_fit")
filename = get_output_path_from_conf(config) + ".png"
render_loss_graph(

2
example_fit_anim.py
View File

@ -8,7 +8,7 @@ from renderer import *
from utils.general import rename_files, get_new_filename
START_IDX = 1 # starting index of the frame to optimize for
FINISH_IDX = 200 # choose a big number to optimize for all frames in samples directory
FINISH_IDX = 60 # choose a big number to optimize for all frames in samples directory
# if False, only run already saved animation without optimization
RUN_OPTIMIZATION = True

51
example_fit_temporal.py Normal file
View File

@ -0,0 +1,51 @@
import os
from utils.video import make_video_with_pip
from utils.graphs import render_loss_graph
from train import optimize_sample
# local imports
from utils.general import get_output_path_from_conf, load_config
from dataset import SMPLyDataset
# load and select sample
config = load_config()
dataset = SMPLyDataset.from_config(config=config)
sample_index = 55
save_to_video = True
if os.getenv('SAMPLE_INDEX') is not None:
sample_index = int(os.getenv('SAMPLE_INDEX'))
# train for pose
best_pose, camera_transformation, loss_history, step_imgs, loss_components = optimize_sample(
sample_index - 1,
dataset,
config,
interactive=False
)
config['pose']['lr'] = config['pose']['temporal']['lr']
config['pose']['iterations'] = config['pose']['temporal']['iterations']
# reuse model to train new sample
pose_temp, camera_transformation, loss_history, step_imgs, loss_components = optimize_sample(
sample_index,
dataset,
config,
interactive=False,
offscreen=True,
initial_pose=best_pose.body_pose.detach().clone().cpu(),
initial_orient=best_pose.global_orient.detach().clone().cpu()
)
if save_to_video:
img_path = dataset.get_image_path(sample_index)
make_video_with_pip(step_imgs, pip_image_path=img_path,
video_name="example_fit_temporal")
filename = get_output_path_from_conf(config) + ".png"
render_loss_graph(
loss_history=loss_history,
loss_components=loss_components,
save=True,
filename=filename)

24
example_hyper.py → example_hyper_optim.py
View File

@ -12,8 +12,8 @@ from utils.general import *
from renderer import *
from utils.general import rename_files, get_new_filename
START_IDX = 00 # starting index of the frame to optimize for
FINISH_IDX = 20 # choose a big number to optimize for all frames in samples
START_IDX = 140 # starting index of the frame to optimize for
FINISH_IDX = 300 # choose a big number to optimize for all frames in samples
device = torch.device('cpu')
dtype = torch.float32
@ -31,7 +31,7 @@ def run_test(config):
FINISH_IDX,
verbose=False,
offscreen=False,
save_to_file=False,
save_to_file=True,
interpolate=False
)
video_name = getfilename_from_conf(
@ -49,24 +49,28 @@ def run_test(config):
save_to_video(
model_outs, video_name, config,
start_frame_offset=START_IDX,
dataset=dataset, interpolation_target=60
)
def run_pose_tests(config):
priors_types = ['bodyPrior', 'anglePrior', 'angleSumLoss', 'temporal']
priors_types = ['bodyPrior', 'anglePrior',
'angleSumLoss', 'temporal', 'intersectLoss', 'changeLoss']
l = [False, True]
permutations = [list(i)
for i in itertools.product(l, repeat=len(priors_types))]
lr_steps = [0.01, 0.02, 0.03, 0.04, 0.05,
0.07, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5]
lr_steps = [0.01]
total_runs = len(permutations) * len(lr_steps)
run_num = 1
for lr in lr_steps:
print("running test with lr:", lr)
config['pose']['lr'] = lr
for p in permutations:
print("running test:", config['pose']['optimizer'])
print("running test ", "(" + str(run_num) + "/" + str(total_runs) + "):",
config['pose']['optimizer'])
# iterate over all permutations and update config
for i, v in enumerate(p):
config['pose'][priors_types[i]]['enabled'] = v
@ -79,7 +83,7 @@ print("training: Adam")
# run tests for adam
run_pose_tests(config)
print("training: LBFGS")
# print("training: LBFGS")
# try the same with lbfgs
config = load_config("./config.lbfgs.temporal.yaml")
run_pose_tests(config)
# config = load_config("./config.lbfgs.temporal.yaml")
# run_pose_tests(config)

11
example_render_video.py
View File

@ -8,8 +8,8 @@ from utils.general import *
from renderer import *
from utils.general import rename_files, get_new_filename
START_IDX = 1 # starting index of the frame to optimize for
FINISH_IDX = None # choose a big number to optimize for all frames in samples directory
START_IDX = 150 # starting index of the frame to optimize for
FINISH_IDX = 300 # choose a big number to optimize for all frames in samples directory
result_image = []
idx = START_IDX
@ -24,13 +24,14 @@ model_outs, filename = create_animation(
START_IDX,
FINISH_IDX,
verbose=False,
offscreen=True,
offscreen=False,
save_to_file=False,
interpolate=False
)
video_name = getfilename_from_conf(
video_name = get_output_path_from_conf(
config) + "-" + str(START_IDX) + "-" + str(FINISH_IDX)
save_to_video(model_outs, video_name, config,
dataset=dataset, interpolation_target=60)
start_frame_offset=START_IDX,
dataset=dataset)

0
example_temporal.py → example_temporal_optim.py
View File

18
train.py
View File

@ -18,12 +18,12 @@ from utils.video import interpolate_poses
from camera_estimation import TorchCameraEstimate
def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), dtype=torch.float32, interactive=True, offscreen=False, verbose=True, initial_pose=None):
def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), dtype=torch.float32, interactive=True, offscreen=False, verbose=True, initial_pose=None, initial_orient=None):
# prepare data and SMPL model
model = SMPLyModel.model_from_conf(config)
init_keypoints, init_joints, keypoints, conf, est_scale, r, img_path = setup_training(
model=model,
renderer=interactive,
renderer=(interactive or offscreen),
dataset=dataset,
sample_index=sample_index,
offscreen=offscreen
@ -69,8 +69,9 @@ def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), d
params = defaultdict(
body_pose=initial_pose,
global_orient=initial_orient
)
with torch.no_grad():
model(**params)
# apply transform to scene
@ -88,6 +89,11 @@ def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), d
render_steps=(offscreen or interactive)
)
# FIXME: there seems to some form of projection issue and hence the orientation is missestimating the angle
# with torch.no_grad():
# .to(device=device, dtype=dtype)
# model.global_orient[0][1] = -model.global_orient[0][1]
# train for pose
best_out, loss_history, step_imgs, loss_components = train_pose_with_conf(
config=config,
@ -118,6 +124,7 @@ def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False
end_idx = len(dataset) - 1
initial_pose = None
initial_orient = None
for idx in trange(end_idx - start_idx, desc='Optimizing'):
idx = start_idx + idx
@ -133,8 +140,8 @@ def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False
verbose=verbose,
offscreen=offscreen,
interactive=verbose,
initial_pose=initial_pose
)
initial_pose=initial_pose,
initial_orient=initial_orient)
if verbose:
print("Optimization of", idx, "frames finished")
@ -152,6 +159,7 @@ def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False
if use_temporal_data:
initial_pose = best_out.body_pose.detach().clone().cpu() # .to(device=device)
initial_orient = best_out.global_orient.detach().clone().cpu()
if interpolate:
model_outs = interpolate_poses(model_outs)

2
train_orient.py
View File

@ -176,4 +176,4 @@ def train_orient_with_conf(
use_progress_bar=use_progress_bar,
)
return best_output.global_orient
return best_output.global_orient.detach().clone().cpu()

40
utils/video.py
View File

@ -48,7 +48,8 @@ def save_to_video(
config: object,
fps=30,
include_thumbnail=True,
thumbnail_size=0.25,
thumbnail_size=0.2,
start_frame_offset=0,
dataset: SMPLyDataset = None,
interpolation_target=None
):
@ -76,9 +77,14 @@ def save_to_video(
inter_ratio = int(interpolation_target / fps)
num_intermediate = inter_ratio - 1
sample_output = interpolate_poses(sample_output, num_intermediate)
else:
sample_output = [
(
out.vertices.detach().cpu().numpy()[0],
cam
) for out, cam in sample_output]
frames = []
print("[export] rendering animation frames...")
print("[export] rendering animation frames...", sample_output[0][0].shape)
# just use the first transform
cam_transform = sample_output[0][1]
@ -87,7 +93,7 @@ def save_to_video(
r.render_model_geometry(
faces=model_anim.faces,
vertices=vertices,
pose=cam_transform,
pose=cam_trans # cam_transform,
)
frames.append(r.get_snapshot())
@ -98,6 +104,8 @@ def save_to_video(
def post_process_frame(img, idx: int):
if not include_thumbnail:
return img
# account for start from frames not zero
idx = start_frame_offset + idx
frame_idx = idx
if interpolation_target is not None:
# account for possible interpolation
@ -122,6 +130,30 @@ def save_to_video(
post_process_frame=post_process_frame)
def make_video_with_pip(frames, pip_image_path, video_name: str, fps=30, ext: str = "mp4", image_size=0.2):
"""renders a video with a pip frame in the corner
"""
def post_process_frame(img, idx: int):
overlay = cv2.imread(pip_image_path)
if overlay is None:
print("[error] image could not be ", pip_image_path)
return img
overlay = cv2.resize(
overlay,
dsize=(
int(overlay.shape[1] * image_size),
int(overlay.shape[0] * image_size)
))
img[0:overlay.shape[0], 0:overlay.shape[1]] = overlay
return img
make_video(frames, video_name, fps,
post_process_frame=post_process_frame)
def interpolate_poses(poses, num_intermediate=5):
"""
Interpolate vertices and cameras between pairs of frames by adding intermediate results