diff --git a/.gitignore b/.gitignore index d864d9e..e73c965 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -/zig-cache/ -/zig-out/ +zig-cache/ +zig-out/ diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 0000000..091522a --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,44 @@ +zig = zig +zig_version != $(zig) version + +.PHONY: clean zig-version + +default: debug + +zig-version: + @echo zig-$(zig_version) + +all: wasm linux macos windows + +debug: zig-version + $(zig) build -Doptimize=Debug + +native: zig-version + $(zig) build -Doptimize=ReleaseFast + +wasm: zig-version + $(zig) build -Dtarget=wasm32-freestanding -Doptimize=ReleaseSmall + +wasm-debug: zig-version + $(zig) build -Dtarget=wasm32-freestanding -Doptimize=Debug + +linux: zig-version + $(zig) build -Dtarget=x86_64-linux-gnu -Doptimize=ReleaseFast + +macos: zig-version + $(zig) build -Dtarget=aarch64-macos-none -Doptimize=ReleaseFast + +windows: zig-version + $(zig) build -Dtarget=x86_64-windows-msvc -Doptimize=ReleaseFast + +test: zig-version + $(zig) build test --summary all + +clean: + rm -rf zig-out zig-cache + +serve: wasm + python -m http.server 8000 -b 127.0.0.1 -d zig-out + +serve-debug: wasm-debug + python -m http.server 8000 -b 127.0.0.1 -d zig-out diff --git a/examples/build.zig b/examples/build.zig new file mode 100644 index 0000000..67b068a --- /dev/null +++ b/examples/build.zig @@ -0,0 +1,77 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + _ = buildModule(b, "face_alignment", target, optimize); + + const fmt_step = b.step("fmt", "Run zig fmt"); + const fmt = b.addFmt(.{ + .paths = &.{ "src", "build.zig", "build.zig.zon" }, + .check = true, + }); + fmt_step.dependOn(&fmt.step); + b.default_step.dependOn(fmt_step); +} + +fn buildModule( + b: *std.Build, + name: []const u8, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, +) *std.Build.Step.Compile { + const zignal = b.dependency("zignal", .{ .target = target, .optimize = optimize }); + var module: *std.Build.Step.Compile = undefined; + + if (target.result.isWasm()) { + module = b.addExecutable(.{ + .name = name, + .root_source_file = .{ .path = b.fmt("src/{s}.zig", .{name}) }, + .optimize = optimize, + .target = b.resolveTargetQuery(.{ + .cpu_arch = .wasm32, + .os_tag = .freestanding, + .cpu_features_add = std.Target.wasm.featureSet(&.{ + .atomics, + .bulk_memory, + // .extended_const, not supported by Safari + .multivalue, + .mutable_globals, + .nontrapping_fptoint, + .reference_types, + //.relaxed_simd, not supported by Firefox or Safari + .sign_ext, + .simd128, + // .tail_call, not supported by Safari + }), + }), + }); + module.entry = .disabled; + module.use_llvm = true; + module.use_lld = true; + // Install files in the .prefix (zig-out) directory + b.getInstallStep().dependOn( + &b.addInstallFile( + module.getEmittedBin(), + b.fmt("{s}.wasm", .{name}), + ).step, + ); + b.installDirectory(.{ + .source_dir = .{ .path = "lib" }, + .install_dir = .prefix, + .install_subdir = "", + }); + } else { + module = b.addSharedLibrary(.{ + .name = name, + .root_source_file = .{ .path = b.fmt("src/{s}.zig", .{name}) }, + .target = target, + .optimize = optimize, + }); + module.root_module.strip = optimize != .Debug and target.result.os.tag != .windows; + b.installArtifact(module); + } + module.rdynamic = true; + module.root_module.addImport("zignal", zignal.module("zignal")); + return module; +} diff --git a/examples/build.zig.zon b/examples/build.zig.zon new file mode 100644 index 0000000..d6a8b1b --- /dev/null +++ b/examples/build.zig.zon @@ -0,0 +1,18 @@ +.{ + .name = "zignal-examples", + .version = "0.0.0", + + .dependencies = .{ + .zignal = .{ + .url = "https://github.com/bfactory-ai/zignal/archive/7cd7e70415dec093eb1b096218d2286d9f829360.tar.gz", + .hash = "1220140e8744befa3cfe2c844d73e62376425eb6dd54140787218b4a480001219433", + }, + }, + + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + "lib", + }, +} diff --git a/examples/lib/face-alignment.html b/examples/lib/face-alignment.html new file mode 100644 index 0000000..710ef96 --- /dev/null +++ b/examples/lib/face-alignment.html @@ -0,0 +1,38 @@ + + + + + + + + Face Alignment + + + + + +
+ + +
+ Click start or drop an image +
+ +
+ + +
+ + + +
+
+

+

+
+
+ +
+ + + diff --git a/examples/lib/face-alignment.js b/examples/lib/face-alignment.js new file mode 100644 index 0000000..95dac38 --- /dev/null +++ b/examples/lib/face-alignment.js @@ -0,0 +1,273 @@ +(function () { + async function setupMediaPipeLandmarks(mode, delegate) { + const mediapipeVersion = "0.10.11"; + const visionBundle = await import( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@" + mediapipeVersion + ); + const { FaceLandmarker, FilesetResolver } = visionBundle; + const vision = await FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@" + + mediapipeVersion + + "/wasm", + ); + return await FaceLandmarker.createFromOptions(vision, { + baseOptions: { + modelAssetPath: + "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", + delegate: delegate, + }, + runningMode: mode, + numFaces: 2, + }); + } + + const video = document.getElementById("video"); + const canvas1 = document.getElementById("canvas1"); + const canvas2 = document.getElementById("canvas2"); + const ctx1 = canvas1.getContext("2d", { willReadFrequently: true }); + const ctx2 = canvas2.getContext("2d", { willReadFrequently: true }); + const toggleButton = document.getElementById("toggle-button"); + const alignButton = document.getElementById("align-button"); + let mediaStream = undefined; + let faceLandmarker = undefined; + let makeup = undefined; + let processFn = undefined; + let image = undefined; + let original = undefined; + let padding = 25; + + document.getElementsByName("padding")[0].innerHTML = padding + '%'; + document.getElementsByName("padding-range")[0].oninput = function () { + padding = this.value; + document.getElementsByName("padding")[0].innerHTML = padding + '%'; + }; + + function displayImageSize() { + let sizeElement = document.getElementById("size"); + sizeElement.textContent = + "size: " + canvas1.width + "×" + canvas1.height + " px."; + } + + toggleButton.disabled = true; + toggleButton.addEventListener("click", () => { + if (mediaStream) { + stopMediaStream(); + } else { + startMediaStream(); + } + }); + + alignButton.disabled = true; + alignButton.addEventListener("click", () => { + processFn(); + }); + + canvas1.ondragover = function (event) { + event.preventDefault(); + }; + + canvas1.ondrop = function (event) { + event.preventDefault(); + let img = new Image(); + img.onload = function () { + canvas1.width = img.width; + canvas1.height = img.height; + ctx1.drawImage(img, 0, 0); + image = ctx1.getImageData(0, 0, canvas1.width, canvas1.height); + original = new ImageData(image.width, image.height); + original.data.set(image.data); + displayImageSize(); + }; + img.src = URL.createObjectURL(event.dataTransfer.files[0]); + alignButton.disabled = false; + }; + + function displayImage(file) { + const reader = new FileReader(); + reader.onload = function (e) { + const imageData = e.target.result; + const img = document.createElement("img"); + img.src = imageData; + img.onload = function () { + canvas1.width = img.width; + canvas1.height = img.height; + ctx1.drawImage(img, 0, 0); + image = ctx1.getImageData(0, 0, canvas1.width, canvas1.height); + original = new ImageData(image.width, image.height); + original.data.set(image.data); + displayImageSize(); + }; + }; + reader.readAsDataURL(file); + } + + const fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.style.display = "none"; + fileInput.addEventListener("change", function (e) { + const file = e.target.files[0]; + displayImage(file); + alignButton.disabled = false; + }); + + canvas1.addEventListener("click", function () { + fileInput.click(); + }); + + function startMediaStream() { + function loop() { + if (!mediaStream) return; + ctx1.save(); + ctx1.scale(-1, 1); + ctx1.drawImage(video, 0, 0, -canvas1.width, canvas1.height); + processFn(); + ctx1.restore(); + alignButton.disabled = true; + requestAnimationFrame(loop); + } + + toggleButton.textContent = "Stop"; + navigator.mediaDevices + .getUserMedia({ video: true }) + .then((stream) => { + mediaStream = stream; + video.srcObject = stream; + video.style.display = "none"; + video.play(); + video.onloadedmetadata = () => { + canvas1.width = video.videoWidth; + canvas1.height = video.videoHeight; + loop(processFn); + }; + }) + .catch((error) => { + console.error("Error accessing webcam:", error); + }); + } + + function stopMediaStream() { + toggleButton.textContent = "Start"; + if (mediaStream) { + mediaStream.getTracks().forEach((track) => track.stop()); + mediaStream = null; + video.srcObject = null; + ctx1.clearRect(0, 0, canvas1.width, canvas1.height); + ctx2.clearRect(0, 0, canvas2.width, canvas2.height); + } + } + + let wasm_promise = fetch("face_alignment.wasm"); + var wasm_exports = null; + const text_decoder = new TextDecoder(); + const text_encoder = new TextEncoder(); + + function decodeString(ptr, len) { + if (len === 0) return ""; + return text_decoder.decode( + new Uint8Array(wasm_exports.memory.buffer, ptr, len), + ); + } + + function unwrapString(bigint) { + const ptr = Number(bigint & 0xffffffffn); + const len = Number(bigint >> 32n); + return decodeString(ptr, len); + } + + WebAssembly.instantiateStreaming(wasm_promise, { + js: { + log: function (ptr, len) { + const msg = decodeString(ptr, len); + console.log(msg); + }, + now: function () { + return performance.now(); + }, + }, + }).then(function (obj) { + wasm_exports = obj.instance.exports; + window.wasm = obj; + setupMediaPipeLandmarks("IMAGE", "GPU").then(function (landmarker) { + faceLandmarker = landmarker; + toggleButton.disabled = false; + + let align = function () { + displayImageSize(); + const rows = canvas1.height; + const cols = canvas1.width; + const numLandmarks = 478; // MediaPipe's landmarks + const landmarksSize = numLandmarks * 2 * 4; // x, y, f32 + const rgbaSize = rows * cols * 4; // RGBA + const extraSize = rgbaSize * 8; // For extra WASM + const outRows = canvas2.height; + const outCols = canvas2.width; + const outSize = outRows * outCols * 4; // RGBA + + // We need to allocate all memory at once before mapping it + const rgbaPtr = wasm_exports.alloc(rgbaSize); + const outPtr = wasm_exports.alloc(outSize); + const landmarksPtr = wasm_exports.alloc(landmarksSize); + const extraPtr = wasm_exports.alloc(extraSize); + // Now we can proceed to map all the memory to JavaScript + let rgba = new Uint8ClampedArray( + wasm_exports.memory.buffer, + rgbaPtr, + rgbaSize, + ); + let landmarks = new Float32Array( + wasm_exports.memory.buffer, + landmarksPtr, + numLandmarks * 2, + ); + image = ctx1.getImageData(0, 0, cols, rows); + rgba.set(image.data); + + const faceLandmarks = faceLandmarker.detect(image).faceLandmarks; + if (faceLandmarks.length === 0) { + ctx2.clearRect(0, 0, canvas2.width, canvas2.height); + return; + } + + // fill the landmarks + for (let i = 0; i < faceLandmarks[0].length; ++i) { + landmarks[i * 2 + 0] = faceLandmarks[0][i].x; + landmarks[i * 2 + 1] = faceLandmarks[0][i].y; + } + + const startTime = performance.now(); + wasm_exports.extract_aligned_face( + rgbaPtr, + rows, + cols, + outPtr, + outRows, + outCols, + padding / 100, + landmarksPtr, + numLandmarks, + extraPtr, + extraSize, + ); + const timeMs = performance.now() - startTime; + const fps = 1000.0 / timeMs; + let timeElement = document.getElementById("time"); + timeElement.textContent = + "time: " + timeMs.toFixed(0) + " ms (" + fps.toFixed(2) + " fps)"; + + let outImg = new Uint8ClampedArray( + wasm_exports.memory.buffer, + outPtr, + outSize, + ); + const out = ctx2.getImageData(0, 0, outCols, outRows); + out.data.set(outImg); + ctx2.putImageData(out, 0, 0); + wasm_exports.free(rgbaPtr, rgbaSize); + wasm_exports.free(outPtr, outSize); + wasm_exports.free(landmarksPtr, landmarksSize); + wasm_exports.free(extraPtr, extraSize); + }; + processFn = align; + }); + }); +})(); diff --git a/examples/index.html b/examples/lib/index.html similarity index 61% rename from examples/index.html rename to examples/lib/index.html index 1086dd2..223bcf0 100644 --- a/examples/index.html +++ b/examples/lib/index.html @@ -5,14 +5,17 @@ Zignal Examples - - +

Zignal Examples

+

Examples

+
- + diff --git a/examples/main.js b/examples/lib/main.js similarity index 100% rename from examples/main.js rename to examples/lib/main.js diff --git a/examples/lib/styles.css b/examples/lib/styles.css new file mode 100644 index 0000000..66658ab --- /dev/null +++ b/examples/lib/styles.css @@ -0,0 +1,39 @@ +#image-container { + display: flex; + margin: 20px; + position: relative; + align-items: center; +} + +#settings-container { + margin: 20px; + position: relative; +} + +.grid-container { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + grid-gap: 20px; +} + +#form { + width: 300px; + margin: 10px; +} + +#canvas1 { + border: 1px dashed black; + vertical-align: center; + width: 640px; + height: auto; + left: 0px; + top: 0px; +} + +#video { + position: absolute; +} + +#toggle-button { + margin: 20px; +} diff --git a/examples/src/face_alignment.zig b/examples/src/face_alignment.zig new file mode 100644 index 0000000..a92b18a --- /dev/null +++ b/examples/src/face_alignment.zig @@ -0,0 +1,127 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const assert = std.debug.assert; +const zignal = @import("zignal"); +const Point2d = zignal.Point2d(f32); +const Image = zignal.Image; +const SimilarityTransform = zignal.SimilarityTransform(f32); +const Rectangle = zignal.Rectangle(f32); +const Rgba = zignal.Rgba; + +pub const std_options: std.Options = .{ + .logFn = if (builtin.cpu.arch.isWasm()) @import("js.zig").logFn else std.log.defaultLog, + .log_level = std.log.default_level, +}; + +/// These landmarks correspond to the closest to dlib's 5 alignment landmarks. +pub const alignment: []const usize = &.{ 263, 398, 33, 173, 2 }; + +/// Extracts the aligned face contained in image using landmarks. +pub fn extractAlignedFace( + comptime T: type, + allocator: std.mem.Allocator, + image: Image(T), + landmarks: []const Point2d, + padding: f32, + out: *Image(T), +) !void { + var from_points: [5]Point2d = .{ + .{ .x = 0.8595674595992, .y = 0.2134981538014 }, + .{ .x = 0.6460604764104, .y = 0.2289674387677 }, + .{ .x = 0.1205750620789, .y = 0.2137274526848 }, + .{ .x = 0.3340850613712, .y = 0.2290642403242 }, + .{ .x = 0.4901123135679, .y = 0.6277975316475 }, + }; + + const to_points: [5]Point2d = .{ + landmarks[alignment[0]].scale(image.cols, image.rows), + landmarks[alignment[1]].scale(image.cols, image.rows), + landmarks[alignment[2]].scale(image.cols, image.rows), + landmarks[alignment[3]].scale(image.cols, image.rows), + landmarks[alignment[4]].scale(image.cols, image.rows), + }; + assert(from_points.len == to_points.len); + assert(out.cols == out.rows); + assert(out.cols > 0); + const size: f32 = @floatFromInt(out.cols); + for (&from_points) |*p| { + p.x = (padding + p.x) / (2 * padding + 1) * size; + p.y = (padding + p.y) / (2 * padding + 1) * size; + } + const transform = SimilarityTransform.find(&from_points, &to_points); + var p = transform.project(.{ .x = 1, .y = 0 }); + p.x -= transform.bias.at(0, 0); + p.y -= transform.bias.at(1, 0); + const angle = std.math.atan2(p.y, p.x); + const scale = @sqrt(p.x * p.x + p.y * p.y); + const center = transform.project(.{ .x = size / 2, .y = size / 2 }); + var rotated = try image.rotateFrom(allocator, center.x, center.y, angle); + defer rotated.deinit(allocator); + + const rect = Rectangle.initCenter(center.x, center.y, size * scale, size * scale); + const crop_top: isize = @intFromFloat(@round(rect.t)); + const crop_left: isize = @intFromFloat(@round(rect.l)); + const crop_rows: usize = @intFromFloat(@round(rect.height())); + const crop_cols: usize = @intFromFloat(@round(rect.width())); + var crop = try Image(T).initAlloc(allocator, crop_rows, crop_cols); + defer crop.deinit(allocator); + for (0..crop_rows) |r| { + const ir: isize = @intCast(r); + for (0..crop_cols) |c| { + const ic: isize = @intCast(c); + crop.data[r * crop_cols + c] = if (rotated.at(@intCast(ir + crop_top), @intCast(ic + crop_left))) |val| + val.* + else + .{ .r = 0, .g = 0, .b = 0, .a = 0 }; + } + } + crop.resize(out); +} + +pub export fn extract_aligned_face( + rgba_ptr: [*]Rgba, + rows: usize, + cols: usize, + out_ptr: [*]Rgba, + out_rows: usize, + out_cols: usize, + padding: f32, + landmarks_ptr: [*]const Point2d, + landmarks_len: usize, + extra_ptr: [*]u8, + extra_size: usize, +) void { + const allocator = blk: { + if (builtin.cpu.arch == .wasm32) { + var fba = std.heap.FixedBufferAllocator.init(extra_ptr[0..extra_size]); + break :blk fba.allocator(); + } else { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + break :blk gpa.allocator(); + } + }; + + const image: Image(Rgba) = .{ + .rows = rows, + .cols = cols, + .data = rgba_ptr[0 .. rows * cols], + }; + std.log.debug("rows: {}, cols: {}", .{ rows, cols }); + + const landmarks: []Point2d = blk: { + var array = std.ArrayList(Point2d).init(allocator); + array.resize(landmarks_len) catch @panic("OOM"); + for (array.items, 0..) |*l, i| { + l.* = landmarks_ptr[i]; + } + break :blk array.toOwnedSlice() catch @panic("OOM"); + }; + defer allocator.free(landmarks); + + var aligned: Image(Rgba) = .{ + .rows = out_rows, + .cols = out_cols, + .data = out_ptr[0 .. out_rows * out_cols], + }; + extractAlignedFace(Rgba, allocator, image, landmarks, padding, &aligned) catch @panic("OOM"); +} diff --git a/examples/src/js.zig b/examples/src/js.zig new file mode 100644 index 0000000..bd82bbd --- /dev/null +++ b/examples/src/js.zig @@ -0,0 +1,38 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const assert = std.debug.assert; +const gpa = std.heap.wasm_allocator; + +pub const js = struct { + extern "js" fn log(ptr: [*]const u8, len: usize) void; + extern "js" fn now() i32; +}; + +pub fn nowFn() f32 { + return @floatFromInt(js.now()); +} + +pub fn logFn( + comptime message_level: std.log.Level, + comptime scope: @TypeOf(.enum_literal), + comptime format: []const u8, + args: anytype, +) void { + const level_txt = comptime message_level.asText(); + const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; + var buf: [500]u8 = undefined; + const line = std.fmt.bufPrint(&buf, level_txt ++ prefix2 ++ format, args) catch l: { + buf[buf.len - 3 ..][0..3].* = "...".*; + break :l &buf; + }; + js.log(line.ptr, line.len); +} + +export fn alloc(len: usize) [*]u8 { + const slice = gpa.alloc(u8, len) catch @panic("OOM"); + return slice.ptr; +} + +export fn free(ptr: [*]const u8, len: usize) void { + gpa.free(ptr[0..len]); +} diff --git a/examples/styles.css b/examples/styles.css deleted file mode 100644 index 8b13789..0000000 --- a/examples/styles.css +++ /dev/null @@ -1 +0,0 @@ -