Skip to content

Commit

Permalink
face blendhapes support
Browse files Browse the repository at this point in the history
  • Loading branch information
@newnon
newnon committed Jun 27, 2023
1 parent f811ad3 commit 3de96ca
Show file tree
Hide file tree
Showing 13 changed files with 2,327 additions and 5 deletions.
1,351 changes: 1,351 additions & 0 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/Face Blendshapes.unity
View file

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/Face Blendshapes.unity.meta
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 30 additions & 3 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/FaceMeshGraph.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class FaceMeshGraph : GraphRunner
{
public int maxNumFaces = 1;
public bool refineLandmarks = true;
public bool blenshapes = false;

private float _minDetectionConfidence = 0.5f;
public float minDetectionConfidence
Expand Down Expand Up @@ -56,17 +57,26 @@ public event EventHandler<OutputEventArgs<List<NormalizedRect>>> OnFaceRectsFrom
remove => _faceRectsFromDetectionsStream.RemoveListener(value);
}

public event EventHandler<OutputEventArgs<ClassificationList>> OnFaceClassificationsFromBlendShapesOutput
{
add => _faceBlendShapesStream?.AddListener(value);
remove => _faceBlendShapesStream?.RemoveListener(value);
}

private const string _InputStreamName = "input_video";

private const string _FaceDetectionsStreamName = "face_detections";
private const string _MultiFaceLandmarksStreamName = "multi_face_landmarks";
private const string _FaceRectsFromLandmarksStreamName = "face_rects_from_landmarks";
private const string _FaceRectsFromDetectionsStreamName = "face_rects_from_detections";
private const string _FaceBlendShapesStreamName = "blendshapes";

private OutputStream<DetectionVectorPacket, List<Detection>> _faceDetectionsStream;
private OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>> _multiFaceLandmarksStream;
private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromLandmarksStream;
private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromDetectionsStream;
private OutputStream<ClassificationListPacket, ClassificationList> _faceBlendShapesStream;


public override void StartRun(ImageSource imageSource)
{
Expand All @@ -76,6 +86,8 @@ public override void StartRun(ImageSource imageSource)
_multiFaceLandmarksStream.StartPolling().AssertOk();
_faceRectsFromLandmarksStream.StartPolling().AssertOk();
_faceRectsFromDetectionsStream.StartPolling().AssertOk();
if (blenshapes)
_faceBlendShapesStream.StartPolling().AssertOk();
}
StartRun(BuildSidePacket(imageSource));
}
Expand All @@ -90,6 +102,8 @@ public override void Stop()
_faceRectsFromLandmarksStream = null;
_faceRectsFromDetectionsStream?.Close();
_faceRectsFromDetectionsStream = null;
_faceBlendShapesStream?.Close();
_faceBlendShapesStream = null;
base.Stop();
}

Expand All @@ -99,15 +113,23 @@ public void AddTextureFrameToInputStream(TextureFrame textureFrame)
}

public bool TryGetNext(out List<Detection> faceDetections, out List<NormalizedLandmarkList> multiFaceLandmarks,
out List<NormalizedRect> faceRectsFromLandmarks, out List<NormalizedRect> faceRectsFromDetections, bool allowBlock = true)
out List<NormalizedRect> faceRectsFromLandmarks, out List<NormalizedRect> faceRectsFromDetections, out ClassificationList faceBlendShapes, bool allowBlock = true)
{
var currentTimestampMicrosec = GetCurrentTimestampMicrosec();
var r1 = TryGetNext(_faceDetectionsStream, out faceDetections, allowBlock, currentTimestampMicrosec);
var r2 = TryGetNext(_multiFaceLandmarksStream, out multiFaceLandmarks, allowBlock, currentTimestampMicrosec);
var r3 = TryGetNext(_faceRectsFromLandmarksStream, out faceRectsFromLandmarks, allowBlock, currentTimestampMicrosec);
var r4 = TryGetNext(_faceRectsFromDetectionsStream, out faceRectsFromDetections, allowBlock, currentTimestampMicrosec);

return r1 || r2 || r3 || r4;
if (blenshapes)
{
var r5 = TryGetNext(_faceBlendShapesStream, out faceBlendShapes, allowBlock, currentTimestampMicrosec);
return r1 || r2 || r3 || r4 || r5;
}
else
{
faceBlendShapes = null;
return r1 || r2 || r3 || r4;
}
}

protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
Expand All @@ -122,13 +144,18 @@ protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
calculatorGraph, _FaceRectsFromLandmarksStreamName, config.AddPacketPresenceCalculator(_FaceRectsFromLandmarksStreamName), timeoutMicrosec);
_faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(
calculatorGraph, _FaceRectsFromDetectionsStreamName, config.AddPacketPresenceCalculator(_FaceDetectionsStreamName), timeoutMicrosec);
if (blenshapes)
_faceBlendShapesStream = new OutputStream<ClassificationListPacket, ClassificationList>(
calculatorGraph, _FaceBlendShapesStreamName, config.AddPacketPresenceCalculator(_FaceBlendShapesStreamName), timeoutMicrosec);
}
else
{
_faceDetectionsStream = new OutputStream<DetectionVectorPacket, List<Detection>>(calculatorGraph, _FaceDetectionsStreamName, true, timeoutMicrosec);
_multiFaceLandmarksStream = new OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>>(calculatorGraph, _MultiFaceLandmarksStreamName, true, timeoutMicrosec);
_faceRectsFromLandmarksStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromLandmarksStreamName, true, timeoutMicrosec);
_faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromDetectionsStreamName, true, timeoutMicrosec);
if (blenshapes)
_faceBlendShapesStream = new OutputStream<ClassificationListPacket, ClassificationList>(calculatorGraph, _FaceBlendShapesStreamName, true, timeoutMicrosec);
}

using (var validatedGraphConfig = new ValidatedGraphConfig())
Expand Down
13 changes: 11 additions & 2 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/FaceMeshSolution.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ protected override void OnStartRun()
graphRunner.OnMultiFaceLandmarksOutput += OnMultiFaceLandmarksOutput;
graphRunner.OnFaceRectsFromLandmarksOutput += OnFaceRectsFromLandmarksOutput;
graphRunner.OnFaceRectsFromDetectionsOutput += OnFaceRectsFromDetectionsOutput;
graphRunner.OnFaceClassificationsFromBlendShapesOutput += OnFaceClassificationsFromBlendShapesOutput;
}

var imageSource = ImageSourceProvider.ImageSource;
Expand All @@ -69,20 +70,23 @@ protected override IEnumerator WaitForNextValue()
List<NormalizedLandmarkList> multiFaceLandmarks = null;
List<NormalizedRect> faceRectsFromLandmarks = null;
List<NormalizedRect> faceRectsFromDetections = null;
ClassificationList faceBlendShapes = null;

if (runningMode == RunningMode.Sync)
{
var _ = graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, true);
var _ = graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, out faceBlendShapes, true);
}
else if (runningMode == RunningMode.NonBlockingSync)
{
yield return new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, false));
yield return new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, out faceBlendShapes, false));
}

_faceDetectionsAnnotationController.DrawNow(faceDetections);
_multiFaceLandmarksAnnotationController.DrawNow(multiFaceLandmarks);
_faceRectsFromLandmarksAnnotationController.DrawNow(faceRectsFromLandmarks);
_faceRectsFromDetectionsAnnotationController.DrawNow(faceRectsFromDetections);
if (faceBlendShapes != null)
Debug.Log($"Blendshapes count {faceBlendShapes.Classification.Count}");
}

private void OnFaceDetectionsOutput(object stream, OutputEventArgs<List<Detection>> eventArgs)
Expand All @@ -104,5 +108,10 @@ private void OnFaceRectsFromDetectionsOutput(object stream, OutputEventArgs<List
{
_faceRectsFromDetectionsAnnotationController.DrawLater(eventArgs.value);
}

private void OnFaceClassificationsFromBlendShapesOutput(object stream, OutputEventArgs<ClassificationList> eventArgs)
{
Debug.Log($"Blendshapes count {eventArgs.value?.Classification?.Count}");
}
}
}
Loading

0 comments on commit 3de96ca

Please sign in to comment.