Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

face blendhapes support #935

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+2,331 −6
Open

face blendhapes support #935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,351 changes: 1,351 additions & 0 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/Face Blendshapes.unity
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/Face Blendshapes.unity.meta
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

38 changes: 34 additions & 4 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/FaceMeshGraph.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class FaceMeshGraph : GraphRunner
{
public int maxNumFaces = 1;
public bool refineLandmarks = true;
public bool blenshapes = false;

private float _minDetectionConfidence = 0.5f;
public float minDetectionConfidence
Expand Down Expand Up @@ -56,17 +57,26 @@ public event EventHandler<OutputEventArgs<List<NormalizedRect>>> OnFaceRectsFrom
remove => _faceRectsFromDetectionsStream.RemoveListener(value);
}

public event EventHandler<OutputEventArgs<ClassificationList>> OnFaceClassificationsFromBlendShapesOutput
{
add => _faceBlendShapesStream?.AddListener(value);
remove => _faceBlendShapesStream?.RemoveListener(value);
}

private const string _InputStreamName = "input_video";

private const string _FaceDetectionsStreamName = "face_detections";
private const string _MultiFaceLandmarksStreamName = "multi_face_landmarks";
private const string _FaceRectsFromLandmarksStreamName = "face_rects_from_landmarks";
private const string _FaceRectsFromDetectionsStreamName = "face_rects_from_detections";
private const string _FaceBlendShapesStreamName = "blendshapes";

private OutputStream<DetectionVectorPacket, List<Detection>> _faceDetectionsStream;
private OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>> _multiFaceLandmarksStream;
private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromLandmarksStream;
private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromDetectionsStream;
private OutputStream<ClassificationListPacket, ClassificationList> _faceBlendShapesStream;


public override void StartRun(ImageSource imageSource)
{
Expand All @@ -76,6 +86,8 @@ public override void StartRun(ImageSource imageSource)
_multiFaceLandmarksStream.StartPolling().AssertOk();
_faceRectsFromLandmarksStream.StartPolling().AssertOk();
_faceRectsFromDetectionsStream.StartPolling().AssertOk();
if (blenshapes)
_faceBlendShapesStream.StartPolling().AssertOk();
}
StartRun(BuildSidePacket(imageSource));
}
Expand All @@ -90,6 +102,8 @@ public override void Stop()
_faceRectsFromLandmarksStream = null;
_faceRectsFromDetectionsStream?.Close();
_faceRectsFromDetectionsStream = null;
_faceBlendShapesStream?.Close();
_faceBlendShapesStream = null;
base.Stop();
}

Expand All @@ -99,15 +113,23 @@ public void AddTextureFrameToInputStream(TextureFrame textureFrame)
}

public bool TryGetNext(out List<Detection> faceDetections, out List<NormalizedLandmarkList> multiFaceLandmarks,
out List<NormalizedRect> faceRectsFromLandmarks, out List<NormalizedRect> faceRectsFromDetections, bool allowBlock = true)
out List<NormalizedRect> faceRectsFromLandmarks, out List<NormalizedRect> faceRectsFromDetections, out ClassificationList faceBlendShapes, bool allowBlock = true)
{
var currentTimestampMicrosec = GetCurrentTimestampMicrosec();
var r1 = TryGetNext(_faceDetectionsStream, out faceDetections, allowBlock, currentTimestampMicrosec);
var r2 = TryGetNext(_multiFaceLandmarksStream, out multiFaceLandmarks, allowBlock, currentTimestampMicrosec);
var r3 = TryGetNext(_faceRectsFromLandmarksStream, out faceRectsFromLandmarks, allowBlock, currentTimestampMicrosec);
var r4 = TryGetNext(_faceRectsFromDetectionsStream, out faceRectsFromDetections, allowBlock, currentTimestampMicrosec);

return r1 || r2 || r3 || r4;
if (blenshapes)
{
var r5 = TryGetNext(_faceBlendShapesStream, out faceBlendShapes, allowBlock, currentTimestampMicrosec);
return r1 || r2 || r3 || r4 || r5;
}
else
{
faceBlendShapes = null;
return r1 || r2 || r3 || r4;
}
}

protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
Expand All @@ -122,13 +144,18 @@ protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
calculatorGraph, _FaceRectsFromLandmarksStreamName, config.AddPacketPresenceCalculator(_FaceRectsFromLandmarksStreamName), timeoutMicrosec);
_faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(
calculatorGraph, _FaceRectsFromDetectionsStreamName, config.AddPacketPresenceCalculator(_FaceDetectionsStreamName), timeoutMicrosec);
if (blenshapes)
_faceBlendShapesStream = new OutputStream<ClassificationListPacket, ClassificationList>(
calculatorGraph, _FaceBlendShapesStreamName, config.AddPacketPresenceCalculator(_FaceBlendShapesStreamName), timeoutMicrosec);
}
else
{
_faceDetectionsStream = new OutputStream<DetectionVectorPacket, List<Detection>>(calculatorGraph, _FaceDetectionsStreamName, true, timeoutMicrosec);
_multiFaceLandmarksStream = new OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>>(calculatorGraph, _MultiFaceLandmarksStreamName, true, timeoutMicrosec);
_faceRectsFromLandmarksStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromLandmarksStreamName, true, timeoutMicrosec);
_faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromDetectionsStreamName, true, timeoutMicrosec);
if (blenshapes)
_faceBlendShapesStream = new OutputStream<ClassificationListPacket, ClassificationList>(calculatorGraph, _FaceBlendShapesStreamName, true, timeoutMicrosec);
}

using (var validatedGraphConfig = new ValidatedGraphConfig())
Expand Down Expand Up @@ -174,10 +201,13 @@ protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)

protected override IList<WaitForResult> RequestDependentAssets()
{
return new List<WaitForResult> {
var ret = new List<WaitForResult> {
WaitForAsset("face_detection_short_range.bytes"),
WaitForAsset(refineLandmarks ? "face_landmark_with_attention.bytes" : "face_landmark.bytes"),
};
if (blenshapes)
ret.Add(WaitForAsset("face_blendshapes.bytes"));
return ret;
}

private SidePacket BuildSidePacket(ImageSource imageSource)
Expand Down
13 changes: 11 additions & 2 deletions Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/FaceMeshSolution.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ protected override void OnStartRun()
graphRunner.OnMultiFaceLandmarksOutput += OnMultiFaceLandmarksOutput;
graphRunner.OnFaceRectsFromLandmarksOutput += OnFaceRectsFromLandmarksOutput;
graphRunner.OnFaceRectsFromDetectionsOutput += OnFaceRectsFromDetectionsOutput;
graphRunner.OnFaceClassificationsFromBlendShapesOutput += OnFaceClassificationsFromBlendShapesOutput;
}

var imageSource = ImageSourceProvider.ImageSource;
Expand All @@ -69,20 +70,23 @@ protected override IEnumerator WaitForNextValue()
List<NormalizedLandmarkList> multiFaceLandmarks = null;
List<NormalizedRect> faceRectsFromLandmarks = null;
List<NormalizedRect> faceRectsFromDetections = null;
ClassificationList faceBlendShapes = null;

if (runningMode == RunningMode.Sync)
{
var _ = graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, true);
var _ = graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, out faceBlendShapes, true);
}
else if (runningMode == RunningMode.NonBlockingSync)
{
yield return new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, false));
yield return new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, out faceBlendShapes, false));
}

_faceDetectionsAnnotationController.DrawNow(faceDetections);
_multiFaceLandmarksAnnotationController.DrawNow(multiFaceLandmarks);
_faceRectsFromLandmarksAnnotationController.DrawNow(faceRectsFromLandmarks);
_faceRectsFromDetectionsAnnotationController.DrawNow(faceRectsFromDetections);
if (faceBlendShapes != null)
Debug.Log($"Blendshapes count {faceBlendShapes.Classification.Count}");
}

private void OnFaceDetectionsOutput(object stream, OutputEventArgs<List<Detection>> eventArgs)
Expand All @@ -104,5 +108,10 @@ private void OnFaceRectsFromDetectionsOutput(object stream, OutputEventArgs<List
{
_faceRectsFromDetectionsAnnotationController.DrawLater(eventArgs.value);
}

private void OnFaceClassificationsFromBlendShapesOutput(object stream, OutputEventArgs<ClassificationList> eventArgs)
{
Debug.Log($"Blendshapes count {eventArgs.value?.Classification?.Count}");
}
}
}
Loading