Question about rendering SKEL model with aitviewer-skel

[Kelly510]

Hi! I tried to use headless renderer of aitviewer-skel to render SKEL models frame by frame but found abnormal noise in the rendered frames. The following is an example. Wondering if you know how to realize headless rendering of SKEL? Thanks :)

Here is my code.

# Copyright (C) 2024  MPI IS, Marilyn Keller
import os
import glob
import argparse
import numpy as np
import pickle as pkl

from skel.skel_model import SKEL
from aitviewer.models.smpl import SMPLLayer
from aitviewer.utils import resample_positions
from aitviewer.configuration import CONFIG as C
from aitviewer.headless import HeadlessRenderer
from aitviewer.utils.so3 import resample_rotations
from aitviewer.renderables.skel import SKELSequence
from aitviewer.renderables.smpl import SMPLSequence


if __name__ == '__main__':
    
    parser = argparse.ArgumentParser(description='Visualize a SKEL sequence.')
    parser.add_argument('--data_root', type=str, default='datasets/amass/ACCAD')
    parser.add_argument('--fps', type=int, help='Fps of the sequence', default=120)
    parser.add_argument('-z', '--z-up', help='Use Z-up coordinate system. \
        This is usefull for vizualizing sequences of AMASS that are 90 degree rotated', action='store_true')
    parser.add_argument('--offset', help='Offset the SMPL model to display it beside SKEL.', action='store_true')
    args = parser.parse_args()

    fps_in = args.fps # Fps of the sequence
    fps_out = 30 # Fps at which the sequence will be played back

    for skel_file in glob.glob(os.path.join(args.data_root, 'skel_annots', '*', '*.pkl')):
        smpl_file = skel_file.replace('skel_annots', 'smplh_annots').replace('pkl', 'npz')
        assert os.path.exists(smpl_file)

        if args.offset:
            translation = np.array([-1.0, 0.0, 0.0])
        else:
            translation = None
        
        smpl_data = np.load(smpl_file)
        smpl_layer = SMPLLayer(model_type='smplh', gender=smpl_data['gender'].item(), device=C.device)
        poses_smpl = smpl_data['poses']
        trans_smpl = smpl_data['trans']
        betas_smpl = smpl_data['betas']

        skel_data = pkl.load(open(skel_file, 'rb'))
        gender = skel_data['gender']
        skel_layer = SKEL(model_path=C.skel_models, gender=gender)
        poses_skel = skel_data['poses']
        trans_skel = skel_data['trans']
        betas_skel = skel_data['betas']

        if fps_out is not None:
            fps_in_smpl = smpl_data['mocap_framerate'].tolist()
            if fps_in_smpl != fps_out:
                ps = np.reshape(poses_smpl, [poses_smpl.shape[0], -1, 3])
                ps_new = resample_rotations(ps, fps_in_smpl, fps_out)
                poses_smpl = np.reshape(ps_new, [-1, poses_smpl.shape[1]])
                trans_smpl = resample_positions(trans_smpl, fps_in_smpl, fps_out)
            if fps_in != fps_out:
                betas_skel = resample_positions(betas_skel, fps_in, fps_out)
                poses_skel = resample_positions(poses_skel, fps_in, fps_out)
                trans_skel = resample_positions(trans_skel, fps_in, fps_out)

        i_root_end = 3
        i_body_end = i_root_end + smpl_layer.bm.NUM_BODY_JOINTS * 3
        i_left_hand_end = i_body_end + smpl_layer.bm.NUM_HAND_JOINTS * 3
        i_right_hand_end = i_left_hand_end + smpl_layer.bm.NUM_HAND_JOINTS * 3

        i_beta_end = skel_layer.num_betas

        assert poses_smpl.shape[0] == poses_skel.shape[0]
        r = HeadlessRenderer()
        r.scene.camera.position = np.array([-5, 1.7, 0.0])
        dir_name = skel_file.replace('skel_annots', 'rendered').replace('.pkl', '')
        
        for n in range(poses_smpl.shape[0]):
            smpl_seq = SMPLSequence(poses_body=poses_smpl[n:n+1, i_root_end:i_body_end], 
                                    poses_root=poses_smpl[n:n+1, :i_root_end],
                                    poses_left_hand=poses_smpl[n:n+1, i_body_end:i_left_hand_end],
                                    poses_right_hand=poses_smpl[n:n+1, i_left_hand_end:i_right_hand_end],
                                    smpl_layer=smpl_layer,
                                    betas=betas_smpl[np.newaxis],
                                    trans=trans_smpl[n:n+1],
                                    z_up=args.z_up,
                                    show_joint_angles=True, 
                                    position=translation,
                                    name='SMPL')
            skel_seq = SKELSequence(poses_skel[n:n+1], 
                                    skel_layer=skel_layer, 
                                    betas=betas_skel[n:n+1, :i_beta_end],
                                    trans=trans_skel[n:n+1],
                                    z_up=args.z_up,
                                    device=C.device,
                                    fps=fps_out,
                                    fps_in=fps_in,
                                    poses_type='skel',
                                    is_rigged=True, 
                                    show_joint_angles=True, 
                                    name='SKEL')
            img_path = os.path.join(dir_name, 'img_{:06d}.jpg'.format(n))
            r.scene.add(smpl_seq)
            r.scene.add(skel_seq)
            r.lock_to_node(skel_seq, (2, 0.7, 2), smooth_sigma=5.0)
            print('saving image to', img_path)
            r.save_frame(img_path)
            r.scene.remove(smpl_seq)
            r.scene.remove(skel_seq)
        
        r.save_video(frame_dir=dir_name, video_dir=os.path.dirname(dir_name), output_fps=fps_out)
        r.on_close()
        # exit(0)

[MarilynKeller]

Hi, did you have the same artifacts on all frames? Did you try to save directly the sequence video to see?

Maybe what you are seing is the back of a second camera. Not sure you need

r.scene.camera.position = np.array([-5, 1.7, 0.0])

Since you are using: r.lock_to_node(skel_seq, (2, 0.7, 2), smooth_sigma=5.0)`

So I'd suggest try to delete r.scene.camera.position = np.array([-5, 1.7, 0.0]).