problems about the visualization

[Sherlock1956]

Hi,I am currently working with the GRAB dataset and using Manotorch to visualize hand-object interactions. However, when I try to visualize the hand and object meshes together, I find that they are not aligned correctly and don’t seem to be concatenated properly.
I have made sure to apply the correct transformations to both the hand and object meshes, but the result still shows the meshes separate from each other. Could you please help me identify if there is something I might be missing in the dataset or the code to ensure that the hand and object meshes are correctly aligned and concatenated?
Thanks in advance for your assistance!

import numpy as np
import sys
import os
import trimesh
import torch
from scipy.spatial.transform import Rotation as R
from manotorch.manolayer import ManoLayer, MANOOutput
sys.path.append("/Users/lyx/Downloads/dataset/grab")
from GRAB.tools.utils import parse_npz

hand_data_root = "/Users/lyx/Downloads/dataset/grab/data"
def handle_data(hand_data_folder):
    file_name_list = os.listdir(hand_data_folder)
    for file_name in file_name_list:
        file_path = os.path.join(hand_data_folder, file_name)
        data = parse_npz(file_path)
        rhand_parameter = data['rhand']
        global_orient = rhand_parameter['params']['global_orient']
        transl = torch.from_numpy(rhand_parameter['params']['transl'])
        fullpose = rhand_parameter['params']['fullpose']
        obj_path = os.path.join(hand_data_root, data['object']['object_mesh'])
        obj_mesh = trimesh.load(obj_path)
        obj_transl = data['object']['params']['transl']
        obj_global_orient = data['object']['params']['global_orient']
        selected_frame = np.argmax(data['contact']['object'].sum(1))
        obj_transl = obj_transl[selected_frame]
        obj_global_orient = obj_global_orient[selected_frame]
        fullpose = fullpose[selected_frame]
        global_orient = global_orient[selected_frame]
        transl = transl[selected_frame]
        rotation = R.from_rotvec(obj_global_orient)  
        rotation_matrix = rotation.as_matrix()
        rot_matrix_4 = np.eye(4)
        rot_matrix_4[:3,:3] = rotation_matrix
        rot_matrix_4[:3,3] = obj_transl
        obj_mesh.apply_transform(rot_matrix_4)
        mano_layer = ManoLayer(mano_assets_root="/Users/lyx/Downloads/dataset/OkaInk/image/manotorch/assets/mano", flat_hand_mean=True, center_idx=0)
        hand_faces = mano_layer.get_mano_closed_faces()
        axis_angle = torch.from_numpy(np.concatenate([global_orient, fullpose]))
        mano_out: MANOOutput = mano_layer(
            axis_angle.unsqueeze(0).reshape(1, 48),
            (torch.zeros(10).unsqueeze(0)))
        hand_verts = mano_out.verts.squeeze(0).numpy() + transl.numpy()
        hand_mesh = trimesh.Trimesh(vertices=hand_verts, faces=hand_faces)
        concat_mesh = trimesh.util.concatenate(hand_mesh, obj_mesh)
        concat_mesh.show()
if __name__ == "__main__":
    for i in range(1,11):
        hand_data_folder = os.path.join(hand_data_root,'grab', f"s{i}")
        handle_data(hand_data_folder)

[chzh9311]

Hi Yuxiao, I recently encountered a similar problem and figured it out. In case you're still confused about this, I think the problem originates from the inverted object orientations, which means

rot_matrix_4[:3,:3] = rotation_matrix

should be

rot_matrix_4[:3,:3] = rotation_matrix.T

See #23 .