final preparations

config.yaml

@@ -1,5 +1,5 @@
 output:
-  rootDir: "./tests/20-02-21"
+  rootDir: "./presentation"
   prefix: ""
 smpl:
   modelRootDir: ./models
@@ -9,25 +9,25 @@ smpl:
   useVposerInit: false
 data:
   renameFiles: false
-  rootDir: ./samples/video02
+  rootDir: ./samples/video01
   personId: 0
-  sampleImageFormat: "frame-%%%.png"
+  sampleImageFormat: "input_%%%%%%%%%%%%_rendered.png"
-  sampleNameFormat: "input2_%%%%%%%%%%%%_keypoints.json"
+  sampleNameFormat: "input_%%%%%%%%%%%%_keypoints.json"
   sampleCoords: !!python/tuple [1080, 1080]
 camera:
   lr: 0.001
   patience: 10
   optimizer: Adam
 orientation:
-  lr: 0.01
+  lr: 0.03
-  optimizer: Adam
+  optimizer: LBFGS
-  iterations: 100
+  iterations: 5
   joint_names: ["hip-left", "hip-right", "shoulder-left", "shoulder-right"] # joints to be used for optimization
 pose:
   device: cuda
   lr: 0.01
   optimizer: Adam # currently supported Adam, LBFGS
-  iterations: 100
+  iterations: 150
   useCameraIntrinsics: true
   useOpenPoseConf: true # use openpose confidence to weight L2 distance loss
   bodyMeanLoss:
@@ -35,21 +35,21 @@ pose:
     weight: 0.1
   bodyPrior:
     enabled: true
-    weight: 0.01
+    weight: 0.1
   anglePrior:
     enabled: true
-    weight: 0.2
+    weight: 0.05
     # optional per joint configurations
     angleIdx: [56, 53, 12, 9, 37, 40]
-    directions: [-1, 1, -1, -1, 1, -1]
+    directions: [1, -1, -1, -1, 1, -1]
     # weights per joint
-    weights: [0.4, 0.4, 0.4, 0.4, 0.2, 0.2]
+    weights: [0.2, 0.2, 0.8, 0.8, 0.0, 0.0]
   angleLimitLoss:
     enabled: true
     weight: 0.01
   angleSumLoss:
     enabled: true
-    weight: 0.01
+    weight: 0.001
   intersectLoss:
     enabled: true
     weight: 0.5
@@ -57,14 +57,14 @@ pose:
     sigma: 0.5
   changeLoss:
     enabled: true
-    weight: 0.1
+    weight: 0.07
   confWeights:
     enabled: false
   vposerPath: "./vposer_v1_0"
   temporal:
     enabled: true
-    iterations: 75
+    iterations: 50
-    lr: 0.01
+    lr: 0.03
 preview:
   enable: true,
   keypoins:

config_loss_off.yaml

@@ -0,0 +1,81 @@
+output:
+  rootDir: "./tests/final-03"
+  prefix: ""
+smpl:
+  modelRootDir: ./models
+  type: smplx # possible options smplx and smpl
+  gender: MALE # possible options MALE, FEMALE, NEUTRAL
+  ext: npz
+  useVposerInit: false
+data:
+  renameFiles: false
+  rootDir: ./samples/video01
+  personId: 0
+  sampleImageFormat: "input_%%%%%%%%%%%%_rendered.png"
+  sampleNameFormat: "input_%%%%%%%%%%%%_keypoints.json"
+  sampleCoords: !!python/tuple [1080, 1080]
+camera:
+  lr: 0.001
+  patience: 10
+  optimizer: Adam
+orientation:
+  lr: 0.03
+  optimizer: LBFGS
+  iterations: 5
+  joint_names: ["hip-left", "hip-right", "shoulder-left", "shoulder-right"] # joints to be used for optimization
+pose:
+  device: cuda
+  lr: 0.01
+  optimizer: Adam # currently supported Adam, LBFGS
+  iterations: 200
+  useCameraIntrinsics: true
+  useOpenPoseConf: false # use openpose confidence to weight L2 distance loss
+  bodyMeanLoss:
+    enabled: false
+    weight: 0.1
+  bodyPrior:
+    enabled: false
+    weight: 0.01
+  anglePrior:
+    enabled: false
+    weight: 0.2
+    # optional per joint configurations
+    angleIdx: [56, 53, 12, 9, 37, 40]
+    directions: [-1, 1, -1, -1, 1, -1]
+    # weights per joint
+    weights: [0.4, 0.4, 0.4, 0.4, 0.2, 0.2]
+  angleLimitLoss:
+    enabled: false
+    weight: 0.01
+  angleSumLoss:
+    enabled: false
+    weight: 0.001
+  intersectLoss:
+    enabled: false
+    weight: 0.5
+    maxCollisions: 8
+    sigma: 0.5
+  changeLoss:
+    enabled: true
+    weight: 0.2
+  confWeights:
+    enabled: false
+  vposerPath: "./vposer_v1_0"
+  temporal:
+    enabled: true
+    iterations: 30
+    lr: 0.01
+preview:
+  enable: true,
+  keypoins:
+    enable: true,
+    radius: 0.01
+    color: 1.0, 0.0, 1.0, 1.0
+  keypoint_torso:
+    enable: true,
+    radius: 0.01
+    color: 1.0, 0.0, 1.0, 1.0
+  joints:
+    enable: true
+    radius: 0.01
+    color: 0.0, 0.7, 0.0, 1.0

example_fit.py

@@ -1,4 +1,5 @@
 import os
+from utils.video import make_video, make_video_with_pip
 from utils.graphs import render_loss_graph
 from train import optimize_sample
 
@@ -9,19 +10,28 @@ from dataset import SMPLyDataset
 # load and select sample
 config = load_config()
 dataset = SMPLyDataset.from_config(config=config)
-sample_index = 55
+sample_index = 150
+save_to_video = True
+
 
 if os.getenv('SAMPLE_INDEX') is not None:
     sample_index = int(os.getenv('SAMPLE_INDEX'))
 
+
 # train for pose
 pose, camera_transformation, loss_history, step_imgs, loss_components = optimize_sample(
     sample_index,
     dataset,
     config,
-    interactive=True
+    interactive=not save_to_video,
+    offscreen=save_to_video
 )
 
+if save_to_video:
+    img_path = dataset.get_image_path(sample_index)
+    make_video_with_pip(step_imgs, pip_image_path=img_path,
+                        video_name="example_fit")
+
 
 filename = get_output_path_from_conf(config) + ".png"
 render_loss_graph(

example_fit_anim.py

@@ -8,7 +8,7 @@ from renderer import *
 from utils.general import rename_files, get_new_filename
 
 START_IDX = 1  # starting index of the frame to optimize for
-FINISH_IDX = 200   # choose a big number to optimize for all frames in samples directory
+FINISH_IDX = 60   # choose a big number to optimize for all frames in samples directory
 # if False, only run already saved animation without optimization
 RUN_OPTIMIZATION = True
 

example_fit_temporal.py

@@ -0,0 +1,51 @@
+import os
+from utils.video import make_video_with_pip
+from utils.graphs import render_loss_graph
+from train import optimize_sample
+
+# local imports
+from utils.general import get_output_path_from_conf, load_config
+from dataset import SMPLyDataset
+
+# load and select sample
+config = load_config()
+dataset = SMPLyDataset.from_config(config=config)
+sample_index = 55
+save_to_video = True
+
+if os.getenv('SAMPLE_INDEX') is not None:
+    sample_index = int(os.getenv('SAMPLE_INDEX'))
+
+# train for pose
+best_pose, camera_transformation, loss_history, step_imgs, loss_components = optimize_sample(
+    sample_index - 1,
+    dataset,
+    config,
+    interactive=False
+)
+
+config['pose']['lr'] = config['pose']['temporal']['lr']
+config['pose']['iterations'] = config['pose']['temporal']['iterations']
+
+# reuse model to train new sample
+pose_temp, camera_transformation, loss_history, step_imgs, loss_components = optimize_sample(
+    sample_index,
+    dataset,
+    config,
+    interactive=False,
+    offscreen=True,
+    initial_pose=best_pose.body_pose.detach().clone().cpu(),
+    initial_orient=best_pose.global_orient.detach().clone().cpu()
+)
+
+if save_to_video:
+    img_path = dataset.get_image_path(sample_index)
+    make_video_with_pip(step_imgs, pip_image_path=img_path,
+                        video_name="example_fit_temporal")
+
+filename = get_output_path_from_conf(config) + ".png"
+render_loss_graph(
+    loss_history=loss_history,
+    loss_components=loss_components,
+    save=True,
+    filename=filename)

example_hyper_optim.py

@@ -12,8 +12,8 @@ from utils.general import *
 from renderer import *
 from utils.general import rename_files, get_new_filename
 
-START_IDX = 00  # starting index of the frame to optimize for
+START_IDX = 140  # starting index of the frame to optimize for
-FINISH_IDX = 20   # choose a big number to optimize for all frames in samples
+FINISH_IDX = 300   # choose a big number to optimize for all frames in samples
 
 device = torch.device('cpu')
 dtype = torch.float32
@@ -31,7 +31,7 @@ def run_test(config):
         FINISH_IDX,
         verbose=False,
         offscreen=False,
-        save_to_file=False,
+        save_to_file=True,
         interpolate=False
     )
     video_name = getfilename_from_conf(
@@ -49,24 +49,28 @@ def run_test(config):
 
     save_to_video(
         model_outs, video_name, config,
+        start_frame_offset=START_IDX,
         dataset=dataset, interpolation_target=60
     )
 
 
 def run_pose_tests(config):
-    priors_types = ['bodyPrior', 'anglePrior', 'angleSumLoss', 'temporal']
+    priors_types = ['bodyPrior', 'anglePrior',
+                    'angleSumLoss', 'temporal', 'intersectLoss', 'changeLoss']
     l = [False, True]
     permutations = [list(i)
                     for i in itertools.product(l, repeat=len(priors_types))]
 
-    lr_steps = [0.01, 0.02, 0.03, 0.04, 0.05,
+    lr_steps = [0.01]
-                0.07, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5]
+    total_runs = len(permutations) * len(lr_steps)
+    run_num = 1
     for lr in lr_steps:
         print("running test with lr:", lr)
         config['pose']['lr'] = lr
 
         for p in permutations:
-            print("running test:", config['pose']['optimizer'])
+            print("running test ", "(" + str(run_num) + "/" + str(total_runs) + "):",
+                  config['pose']['optimizer'])
             # iterate over all permutations and update config
             for i, v in enumerate(p):
                 config['pose'][priors_types[i]]['enabled'] = v
@@ -79,7 +83,7 @@ print("training: Adam")
 # run tests for adam
 run_pose_tests(config)
 
-print("training: LBFGS")
+# print("training: LBFGS")
 # try the same with lbfgs
-config = load_config("./config.lbfgs.temporal.yaml")
+# config = load_config("./config.lbfgs.temporal.yaml")
-run_pose_tests(config)
+# run_pose_tests(config)

example_render_video.py

@@ -8,8 +8,8 @@ from utils.general import *
 from renderer import *
 from utils.general import rename_files, get_new_filename
 
-START_IDX = 1  # starting index of the frame to optimize for
+START_IDX = 150  # starting index of the frame to optimize for
-FINISH_IDX = None  # choose a big number to optimize for all frames in samples directory
+FINISH_IDX = 300  # choose a big number to optimize for all frames in samples directory
 
 result_image = []
 idx = START_IDX
@@ -24,13 +24,14 @@ model_outs, filename = create_animation(
     START_IDX,
     FINISH_IDX,
     verbose=False,
-    offscreen=True,
+    offscreen=False,
     save_to_file=False,
     interpolate=False
 )
 
-video_name = getfilename_from_conf(
+video_name = get_output_path_from_conf(
     config) + "-" + str(START_IDX) + "-" + str(FINISH_IDX)
 
 save_to_video(model_outs, video_name, config,
-              dataset=dataset, interpolation_target=60)
+              start_frame_offset=START_IDX,
+              dataset=dataset)

train.py

@@ -18,12 +18,12 @@ from utils.video import interpolate_poses
 from camera_estimation import TorchCameraEstimate
 
 
-def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), dtype=torch.float32, interactive=True, offscreen=False, verbose=True, initial_pose=None):
+def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), dtype=torch.float32, interactive=True, offscreen=False, verbose=True, initial_pose=None, initial_orient=None):
     # prepare data and SMPL model
     model = SMPLyModel.model_from_conf(config)
     init_keypoints, init_joints, keypoints, conf, est_scale, r, img_path = setup_training(
         model=model,
-        renderer=interactive,
+        renderer=(interactive or offscreen),
         dataset=dataset,
         sample_index=sample_index,
         offscreen=offscreen
@@ -69,9 +69,10 @@ def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), d
 
     params = defaultdict(
         body_pose=initial_pose,
+        global_orient=initial_orient
     )
-
+    with torch.no_grad():
-    model(**params)
+        model(**params)
 
     # apply transform to scene
     if r is not None:
@@ -88,6 +89,11 @@ def optimize_sample(sample_index, dataset, config, device=torch.device('cpu'), d
         render_steps=(offscreen or interactive)
     )
 
+    # FIXME: there seems to some form of projection issue and hence the orientation is missestimating the angle
+    # with torch.no_grad():
+    # .to(device=device, dtype=dtype)
+    #    model.global_orient[0][1] = -model.global_orient[0][1]
+
     # train for pose
     best_out, loss_history, step_imgs, loss_components = train_pose_with_conf(
         config=config,
@@ -118,6 +124,7 @@ def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False
         end_idx = len(dataset) - 1
 
     initial_pose = None
+    initial_orient = None
 
     for idx in trange(end_idx - start_idx, desc='Optimizing'):
         idx = start_idx + idx
@@ -133,8 +140,8 @@ def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False
             verbose=verbose,
             offscreen=offscreen,
             interactive=verbose,
-            initial_pose=initial_pose
+            initial_pose=initial_pose,
-        )
+            initial_orient=initial_orient)
 
         if verbose:
             print("Optimization of", idx, "frames finished")
@@ -152,6 +159,7 @@ def create_animation(dataset, config, start_idx=0, end_idx=None, offscreen=False
 
         if use_temporal_data:
             initial_pose = best_out.body_pose.detach().clone().cpu()  # .to(device=device)
+            initial_orient = best_out.global_orient.detach().clone().cpu()
 
     if interpolate:
         model_outs = interpolate_poses(model_outs)

train_orient.py

@@ -176,4 +176,4 @@ def train_orient_with_conf(
         use_progress_bar=use_progress_bar,
     )
 
-    return best_output.global_orient
+    return best_output.global_orient.detach().clone().cpu()

utils/video.py

@@ -48,7 +48,8 @@ def save_to_video(
     config: object,
     fps=30,
     include_thumbnail=True,
-    thumbnail_size=0.25,
+    thumbnail_size=0.2,
+    start_frame_offset=0,
     dataset: SMPLyDataset = None,
     interpolation_target=None
 ):
@@ -76,9 +77,14 @@ def save_to_video(
         inter_ratio = int(interpolation_target / fps)
         num_intermediate = inter_ratio - 1
         sample_output = interpolate_poses(sample_output, num_intermediate)
-
+    else:
+        sample_output = [
+            (
+                out.vertices.detach().cpu().numpy()[0],
+                cam
+            ) for out, cam in sample_output]
     frames = []
-    print("[export] rendering animation frames...")
+    print("[export] rendering animation frames...", sample_output[0][0].shape)
 
     # just use the first transform
     cam_transform = sample_output[0][1]
@@ -87,7 +93,7 @@ def save_to_video(
         r.render_model_geometry(
             faces=model_anim.faces,
             vertices=vertices,
-            pose=cam_transform,
+            pose=cam_trans  # cam_transform,
         )
         frames.append(r.get_snapshot())
 
@@ -98,6 +104,8 @@ def save_to_video(
     def post_process_frame(img, idx: int):
         if not include_thumbnail:
             return img
+        # account for start from frames not zero
+        idx = start_frame_offset + idx
         frame_idx = idx
         if interpolation_target is not None:
             # account for possible interpolation
@@ -122,6 +130,30 @@ def save_to_video(
                post_process_frame=post_process_frame)
 
 
+def make_video_with_pip(frames, pip_image_path, video_name: str, fps=30, ext: str = "mp4", image_size=0.2):
+    """renders a video with a pip frame in the corner
+    """
+
+    def post_process_frame(img, idx: int):
+        overlay = cv2.imread(pip_image_path)
+
+        if overlay is None:
+            print("[error] image could not be ", pip_image_path)
+            return img
+
+        overlay = cv2.resize(
+            overlay,
+            dsize=(
+                int(overlay.shape[1] * image_size),
+                int(overlay.shape[0] * image_size)
+            ))
+        img[0:overlay.shape[0], 0:overlay.shape[1]] = overlay
+        return img
+
+    make_video(frames, video_name, fps,
+               post_process_frame=post_process_frame)
+
+
 def interpolate_poses(poses, num_intermediate=5):
     """
     Interpolate vertices and cameras between pairs of frames by adding intermediate results