I am having real hard time integrating kalidokit (face+body tracking)

[melisbsss]

I am trying make kalidokit face and body tracking work but no matter what I did I cannot get the body tracking working. Face tracking works perfectly fine. In similar projects such as kalidokit-react face+body, but I guess since I am trying to make it work through in there is a rendering issue I still can't figure out...however I wonder, then how does the face tracking work? Any help will be appreciated.

AFRAME.registerComponent("m-avatar", {
   schema: {
      src: { type: "string" },
      motion: { type: "string" },
   },
   async init() {
      this.camera = AFRAME.scenes[0].camera;
      this.scene = this.el.object3D;
      this.renderer = AFRAME.scenes[0].renderer;
      var currentVrm = null;
      var currentMixer = null;
      this.modelSrc = this.data.src; //localavatar;
      this.defaultPose = idlePose;
      this.currentClip = null;
      this.vrm = null;
      this.mixer = null;
   },
   async update(oldData) {
      this.camera = AFRAME.scenes[0].camera;
      this.scene = this.el.object3D;
      this.renderer = AFRAME.scenes[0].renderer;
      var currentVrm = null;
      var currentMixer = null;
      var currentClip = null;
      var currentScene = null;
      var currentCamera = null;
      if (this.data.src !== oldData.src) {
         const _vrmData = await loadVRM(this.modelSrc).then(async (vrm) => {
            currentVrm = vrm;
            this.scene.add(currentVrm.scene);
            const head = currentVrm.humanoid.getBoneNode("head");
            this.camera.position.set(0.0, head.getWorldPosition(new THREE.Vector3()).y, 6.0);
            currentMixer = new THREE.AnimationMixer(currentVrm.scene);
            currentMixer.timeScale = 1;
            return await loadMixamoAnimation(this.defaultPose, currentVrm).then(async (clip) => {
               currentClip = currentMixer.clipAction(clip); //.setEffectiveWeight(1.0) doesnwork
               //currentClip.play();
               return [currentVrm, currentMixer, currentClip];
            });
         });
         this.vrm = _vrmData[0];

         this.mixer = _vrmData[1];
         this.currentClip = _vrmData[2];

         const clock = new THREE.Clock();
         clock.start();

         function animate() {
            requestAnimationFrame(animate);
            const delta = clock.getDelta();
            if (currentMixer) {
               currentMixer.update(delta);
            }
            if (currentVrm) {
               currentVrm.update(delta);
            }
         }
         animate();
      }

      if (oldData.motion && this.data.motion !== oldData.motion && this.vrm && this.mixer) {
         if (!this.data.motion.includes("fbx")) {
            // Animate Rotation Helper function
            const rigRotation = (name, rotation = { x: 0, y: 0, z: 0 }, dampener = 1, lerpAmount = 0.3) => {
               if (!this.vrm) {
                  return;
               }
               //const Part = this.vrm.humanoid.getBoneNode(VRMSchema.HumanoidBoneName[name]); //
               const Part = this.vrm.humanoid.getRawBoneNode(VRMHumanBoneName[name]);

               if (!Part) {
                  return;
               }

               let euler = new THREE.Euler(rotation.x * dampener, rotation.y * dampener, rotation.z * dampener, rotation.rotationOrder || "XYZ");
               let quaternion = new THREE.Quaternion().setFromEuler(euler);
               Part.quaternion.slerp(quaternion, lerpAmount); // interpolate
            };

            // Animate Position Helper Function
            const rigPosition = (name, position = { x: 0, y: 0, z: 0 }, dampener = 1, lerpAmount = 0.3) => {
               if (!this.vrm) {
                  return;
               }
               //const Part = this.vrm.humanoid.getBoneNode(VRMSchema.HumanoidBoneName[name]); //
               const Part = this.vrm.humanoid.getRawBoneNode(VRMHumanBoneName[name]);

               if (!Part) {
                  return;
               }
               let vector = new THREE.Vector3(position.x * dampener, position.y * dampener, position.z * dampener);
               Part.position.lerp(vector, lerpAmount); // interpolate
            };

            const rigFace = (riggedFace) => {
               if (!this.vrm) {
                  return;
               }
               //rigRotation("Neck", riggedFace.head, 0.7);

               // Blendshapes and Preset Name Schema
               const Blendshape = this.vrm.expressionManager;
               const PresetName = VRMExpressionPresetName;

               // Simple example without winking. Interpolate based on old blendshape, then stabilize blink with `Kalidokit` helper function.
               // for VRM, 1 is closed, 0 is open.
               riggedFace.eye.l = lerp(clamp(1 - riggedFace.eye.l, 0, 1), Blendshape.getValue(PresetName.Blink), 0.5);
               riggedFace.eye.r = lerp(clamp(1 - riggedFace.eye.r, 0, 1), Blendshape.getValue(PresetName.Blink), 0.5);
               riggedFace.eye = Face.stabilizeBlink(riggedFace.eye, riggedFace.head.y);
               Blendshape.setValue(PresetName.Blink, riggedFace.eye.l);

               // Interpolate and set mouth blendshapes
               Blendshape.setValue(PresetName.Ih, lerp(riggedFace.mouth.shape.I, Blendshape.getValue(PresetName.Ih), 0.5));
               Blendshape.setValue(PresetName.Aa, lerp(riggedFace.mouth.shape.A, Blendshape.getValue(PresetName.Aa), 0.5));
               Blendshape.setValue(PresetName.Ee, lerp(riggedFace.mouth.shape.E, Blendshape.getValue(PresetName.Ee), 0.5));
               Blendshape.setValue(PresetName.Oh, lerp(riggedFace.mouth.shape.O, Blendshape.getValue(PresetName.Oh), 0.5));
               Blendshape.setValue(PresetName.Uu, lerp(riggedFace.mouth.shape.U, Blendshape.getValue(PresetName.Uu), 0.5));

               //PUPILS
               //interpolate pupil and keep a copy of the value
               let lookTarget = new THREE.Euler(lerp(oldLookTarget.x, riggedFace.pupil.y, 0.4), lerp(oldLookTarget.y, riggedFace.pupil.x, 0.4), 0, "XYZ");
               oldLookTarget.copy(lookTarget); 

               const yaw = THREE.MathUtils.RAD2DEG * lookTarget.y;
               const pitch = THREE.MathUtils.RAD2DEG * lookTarget.x;
               this.vrm.lookAt.applier.applyYawPitch(yaw, pitch);
            };

            const clock = new THREE.Clock();
            //clock.start();
            currentVrm = this.vrm;
            currentMixer = this.mixer;

            if (currentVrm && currentMixer) {
               /* VRM Character Animator */
               const animateVRM = async (results) => {
                  // Take the results from `Holistic` and animate character based on its Face, Pose, and Hand Keypoints.
                  let riggedPose, riggedLeftHand, riggedRightHand, riggedFace;

                  const faceLandmarks = results.faceLandmarks;
                  // Pose 3D Landmarks are with respect to Hip distance in meters
                  const pose3DLandmarks = results.za; //results.ea;
                  // Pose 2D landmarks are with respect to videoWidth and videoHeight
                  const pose2DLandmarks = results.poseLandmarks;
                  // Be careful, hand landmarks may be reversed
                  const leftHandLandmarks = results.rightHandLandmarks;
                  const rightHandLandmarks = results.leftHandLandmarks;

                  const videoElement = document.getElementById("video-in");

                  // Animate Face
                  if (faceLandmarks) {
                     riggedFace = await Face.solve(faceLandmarks, {
                        runtime: "tfjs", //mediapipe doesnt work makes the eyes close
                        video: videoElement,
                     });
                     rigFace(riggedFace);
                  }

                  // Animate Pose
                  if (pose2DLandmarks && pose3DLandmarks) {
                     riggedPose = await Pose.solve(pose3DLandmarks, pose2DLandmarks, {
                        runtime: "mediapipe",
                        video: videoElement,
                     }); 

                     rigRotation("Hips", riggedPose.Hips.rotation, 0.7);

                     rigPosition(
                        "Hips",
                        {
                           x: riggedPose.Hips.position.x, // Reverse direction
                           y: riggedPose.Hips.position.y + 1, // Add a bit of height
                           z: -riggedPose.Hips.position.z, // Reverse direction
                        },
                        1,
                        0.07
                     );

                     rigRotation("Chest", riggedPose.Spine, 0.25, 0.3);
                     rigRotation("Spine", riggedPose.Spine, 0.45, 0.3);

                     rigRotation("RightUpperArm", riggedPose.RightUpperArm, 1, 0.3);
                     rigRotation("RightLowerArm", riggedPose.RightLowerArm, 1, 0.3);
                     rigRotation("LeftUpperArm", riggedPose.LeftUpperArm, 1, 0.3);
                     rigRotation("LeftLowerArm", riggedPose.LeftLowerArm, 1, 0.3);

                     rigRotation("LeftUpperLeg", riggedPose.LeftUpperLeg, 1, 0.3);
                     rigRotation("LeftLowerLeg", riggedPose.LeftLowerLeg, 1, 0.3);
                     rigRotation("RightUpperLeg", riggedPose.RightUpperLeg, 1, 0.3);
                     rigRotation("RightLowerLeg", riggedPose.RightLowerLeg, 1, 0.3);
                  }

                  // Animate Hands
                  if (leftHandLandmarks) {
                     riggedLeftHand = await Hand.solve(leftHandLandmarks, "Left");
                     rigRotation("LeftHand", {
                        // Combine pose rotation Z and hand rotation X Y
                        z: riggedPose.LeftHand.z,
                        y: riggedLeftHand.LeftWrist.y,
                        x: riggedLeftHand.LeftWrist.x,
                     });

                     rigRotation("LeftRingProximal", riggedLeftHand.LeftRingProximal);
                     rigRotation("LeftRingIntermediate", riggedLeftHand.LeftRingIntermediate);
                     rigRotation("LeftRingDistal", riggedLeftHand.LeftRingDistal);
                     rigRotation("LeftIndexProximal", riggedLeftHand.LeftIndexProximal);
                     rigRotation("LeftIndexIntermediate", riggedLeftHand.LeftIndexIntermediate);
                     rigRotation("LeftIndexDistal", riggedLeftHand.LeftIndexDistal);
                     rigRotation("LeftMiddleProximal", riggedLeftHand.LeftMiddleProximal);
                     rigRotation("LeftMiddleIntermediate", riggedLeftHand.LeftMiddleIntermediate);
                     rigRotation("LeftMiddleDistal", riggedLeftHand.LeftMiddleDistal);
                     rigRotation("LeftThumbProximal", riggedLeftHand.LeftThumbProximal);
                     rigRotation("LeftThumbIntermediate", riggedLeftHand.LeftThumbIntermediate);
                     rigRotation("LeftThumbDistal", riggedLeftHand.LeftThumbDistal);
                     rigRotation("LeftLittleProximal", riggedLeftHand.LeftLittleProximal);
                     rigRotation("LeftLittleIntermediate", riggedLeftHand.LeftLittleIntermediate);
                     rigRotation("LeftLittleDistal", riggedLeftHand.LeftLittleDistal);
                  }
                  if (rightHandLandmarks) {
                     riggedRightHand = await Hand.solve(rightHandLandmarks, "Right");
                     rigRotation("RightHand", {
                        // Combine Z axis from pose hand and X/Y axis from hand wrist rotation
                        z: riggedPose.RightHand.z,
                        y: riggedRightHand.RightWrist.y,
                        x: riggedRightHand.RightWrist.x,
                     });
                     rigRotation("RightRingProximal", riggedRightHand.RightRingProximal);
                     rigRotation("RightRingIntermediate", riggedRightHand.RightRingIntermediate);
                     rigRotation("RightRingDistal", riggedRightHand.RightRingDistal);
                     rigRotation("RightIndexProximal", riggedRightHand.RightIndexProximal);
                     rigRotation("RightIndexIntermediate", riggedRightHand.RightIndexIntermediate);
                     rigRotation("RightIndexDistal", riggedRightHand.RightIndexDistal);
                     rigRotation("RightMiddleProximal", riggedRightHand.RightMiddleProximal);
                     rigRotation("RightMiddleIntermediate", riggedRightHand.RightMiddleIntermediate);
                     rigRotation("RightMiddleDistal", riggedRightHand.RightMiddleDistal);
                     rigRotation("RightThumbProximal", riggedRightHand.RightThumbProximal);
                     rigRotation("RightThumbIntermediate", riggedRightHand.RightThumbIntermediate);
                     rigRotation("RightThumbDistal", riggedRightHand.RightThumbDistal);
                     rigRotation("RightLittleProximal", riggedRightHand.RightLittleProximal);
                     rigRotation("RightLittleIntermediate", riggedRightHand.RightLittleIntermediate);
                     rigRotation("RightLittleDistal", riggedRightHand.RightLittleDistal);
                  }
               };
               await animateVRM(JSON.parse(this.data.motion));

               const clock = new THREE.Clock();
               function animate() {
                  requestAnimationFrame(animate);
                  const delta = clock.getDelta();
                  if (currentMixer) {
                     currentMixer.update(delta);
                  }
                  if (currentVrm) {
                     currentVrm.update(delta);
                  }
               }
               animate();
            }
         }
      }
   },
});

[ke456-png]

I recommend using the getNormalizedBoneNode(), which is an interface for absorbing differences in local bone axes between models.

const Part = this.vrm.humanoid.getNormalizedBoneNode(VRMHumanBoneName[name]);

If you have a specific intention to use the getRawBoneNode() please disable autoUpdateHumanBones.

this.vrm.humanoid.autoUpdateHumanBones = false;
const Part = this.vrm.humanoid.getRawBoneNode(VRMHumanBoneName[name]);