body-pose-animation/train.py

# library imports
from collections import defaultdict
import numpy as np
from modules.camera import SimpleCamera
from train_orient import train_orient_with_conf
from modules.utils import is_loss_enabled, toggle_loss_enabled
import os
import pickle
import cv2
import torch
from tqdm.auto import trange

# local imports
from train_pose import train_pose_with_conf
from model import SMPLyModel
from utils.general import getfilename_from_conf, setup_training
from utils.video import interpolate_poses
from camera_estimation import TorchCameraEstimate


def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), dtype=torch.float32, interactive=True, offscreen=False, verbose=True, initial_pose=None, initial_orient=None):
    # prepare data and SMPL model
    model = SMPLyModel.model_from_conf(config)
    init_keypoints, init_joints, keypoints, conf, est_scale, r, img_path = setup_training(
        model=model,
        renderer=(interactive or offscreen),
        dataset=dataset,
        sample_index=sample_index,
        offscreen=offscreen
    )

    camera = TorchCameraEstimate(
        model,
        keypoints=keypoints,
        renderer=r,
        device=device,
        dtype=dtype,
        image_path=img_path,
        est_scale=est_scale,
        verbose=verbose,
        use_progress_bar=verbose
    )

    if not offscreen and interactive:
        # render camera to the scene
        camera.setup_visualization(r.init_keypoints, r.keypoints)

    # change loss requires for at least one pass to have completed
    # we disabled this loss if enabled at all for the first pass
    change_loss_disabled = False
    if is_loss_enabled(config, 'changeLoss') and initial_pose is None:
        change_loss_disabled = True
        toggle_loss_enabled(config, 'changeLoss', False)

    # configure PyTorch device and format
    # dtype = torch.float64
    if 'device' in config['pose'] is not None:
        device = torch.device(config['pose']['device'])
    else:
        device = torch.device('cpu')

    # get camera estimation
    pose_camera, cam_trans, cam_int, cam_params = SimpleCamera.from_estimation_cam(
        cam=camera,
        use_intrinsics=config['pose']['useCameraIntrinsics'],
        dtype=dtype,
        device=device,
    )

    params = defaultdict(
        body_pose=initial_pose,
        global_orient=initial_orient
    )
    with torch.no_grad():
        model(**params)

    # apply transform to scene
    if r is not None:
        r.set_group_pose("body", cam_trans.cpu().numpy())

    global_orient = train_orient_with_conf(
        config=config,
        model=model,
        keypoints=keypoints,
        camera_layer=pose_camera,
        renderer=r,
        device=device,
        use_progress_bar=verbose,
        render_steps=(offscreen or interactive)
    )

    # FIXME: there seems to some form of projection issue and hence the orientation is missestimating the angle
    # with torch.no_grad():
    # .to(device=device, dtype=dtype)
    #    model.global_orient[0][1] = -model.global_orient[0][1]

    # train for pose
    best_out, loss_history, step_imgs, loss_components = train_pose_with_conf(
        config=config,
        model=model,
        keypoints=keypoints,
        keypoint_conf=conf,
        pose_camera=pose_camera,
        renderer=r,
        device=device,
        use_progress_bar=verbose,
        render_steps=(offscreen or interactive)
    )

    # make sure change loss is enabled for the next sample
    if change_loss_disabled:
        toggle_loss_enabled(config, 'changeLoss', True)

    # if display_result and interactive:
    #     r.wait_for_close()

    return best_out, cam_trans, loss_history, step_imgs, loss_components


def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False, verbose=False, save_to_file=False, interpolate=False):
    model_outs = []
    use_temporal_data = config['pose']['temporal']['enabled']
    if end_idx is None:
        end_idx = len(dataset) - 1

    initial_pose = None
    initial_orient = None

    for idx in trange(end_idx - start_idx, desc='Optimizing'):
        idx = start_idx + idx

        if use_temporal_data and initial_pose is not None:
            config['pose']['lr'] = config['pose']['temporal']['lr']
            config['pose']['iterations'] = config['pose']['temporal']['iterations']

        best_out, cam_trans, train_loss, step_imgs, loss_components = optimize_sample(
            idx,
            dataset,
            config,
            verbose=verbose,
            offscreen=offscreen,
            interactive=verbose,
            initial_pose=initial_pose,
            initial_orient=initial_orient)

        if verbose:
            print("Optimization of", idx, "frames finished")

        # print("\nPose optimization of frame", idx, "is finished.")
        R = cam_trans.cpu().numpy().squeeze()
        idx += 1

        if best_out is None:
            print("[error] optimizer produced no pose. Skipping frame:", idx)
            continue

        # append optimized pose and camera transformation to the array
        model_outs.append((best_out, R))

        if use_temporal_data:
            initial_pose = best_out.body_pose.detach().clone().cpu()  # .to(device=device)
            initial_orient = best_out.global_orient.detach().clone().cpu()

    if interpolate:
        model_outs = interpolate_poses(model_outs)

    file_path = None

    if save_to_file:
        '''
        Save final_poses array into results folder as a pickle dump
        '''
        results_dir = config['output']['rootDir']
        result_prefix = config['output']['prefix']

        pkl_name = getfilename_from_conf(config)

        pkl_name = pkl_name + "-" + str(start_idx)
        if end_idx is not None:
            pkl_name = pkl_name + "-" + str(end_idx)

        pkl_name = pkl_name + ".pkl"

        file_path = os.path.join(results_dir, pkl_name)
        print("Saving results to", file_path)
        with open(file_path, "wb") as fp:
            pickle.dump(model_outs, fp)
        print("Results have been saved to", file_path)

    return model_outs, file_path