Major x11 screen capture optimization

Author: Neakita

SightKeeper.Application.Linux/Natives/LibXExt.cs

@@ -12,7 +12,7 @@ internal static unsafe partial class LibXExt
 
 	[LibraryImport(DllName)]
 	[return: MarshalAs(UnmanagedType.Bool)]
-	public static partial bool XShmGetImage(IntPtr display, UIntPtr drawable, XImage* image, int x, int y, UIntPtr plane_mask);
+	public static partial bool XShmGetImage(IntPtr display, UIntPtr drawable, XImage* image, int x, int y, ulong plane_mask);
 
 	[LibraryImport(DllName, SetLastError = true)]
 	public static partial int XShmQueryExtension(IntPtr display);

SightKeeper.Application.Linux/SharedImageMemorySegment.cs

@@ -1,3 +1,4 @@
+using System.Numerics.Tensors;
 using System.Runtime.InteropServices;
 using SightKeeper.Application.Linux.Natives;
 using SightKeeper.Domain.Model;
@@ -23,10 +24,14 @@ public unsafe SharedImageMemorySegment(nint display, Vector2<ushort> resolution)
 
 	public unsafe void FetchData(int screen, Vector2<ushort> offset)
 	{
-		const int allPlanes = ~0;
-		UIntPtr allPlanes2 = new(unchecked((uint)allPlanes));
+		const ulong allPlanes = unchecked((ulong)~0);
 		var drawable = (UIntPtr)LibX.XRootWindow(_display, screen);
-		LibXExt.XShmGetImage(_display, drawable, _image.ximage, offset.X, offset.Y, allPlanes2);
+		LibXExt.XShmGetImage(_display, drawable, _image.ximage, offset.X, offset.Y, allPlanes);
+		
+		// xlib doesn't use most significant byte but fills it with zeros
+		// because of that ImageSharp treats it as fully transparent Bgra32
+		Span<uint> span = new(_image.data, Resolution.X * Resolution.Y);
+		TensorPrimitives.BitwiseOr(span, 0xFF_00_00_00, span);
 	}
 
 	public void Dispose()

SightKeeper.Application.Linux/SightKeeper.Application.Linux.csproj

@@ -11,4 +11,8 @@
       <ProjectReference Include="..\SightKeeper.Application\SightKeeper.Application.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="System.Numerics.Tensors" Version="9.0.0-rc.1.24431.7" />
+    </ItemGroup>
+
 </Project>

SightKeeper.Application.Linux/X11ScreenCapture.cs

@@ -1,41 +1,32 @@
 using SightKeeper.Application.Linux.Natives;
 using SightKeeper.Domain.Model;
 using SixLabors.ImageSharp;
-using SixLabors.ImageSharp.Formats;
-using SixLabors.ImageSharp.Formats.Bmp;
 using SixLabors.ImageSharp.PixelFormats;
 
 namespace SightKeeper.Application.Linux;
 
 public sealed class X11ScreenCapture : ScreenCapture, IDisposable
 {
-	private static readonly IImageEncoder Encoder = new BmpEncoder();
-
 	public X11ScreenCapture()
 	{
 		_display = LibX.XOpenDisplay(null);
 		_screen = LibX.XDefaultScreen(_display);
 		if (LibXExt.XShmQueryExtension(_display) == 0)
 		{
-			LibX.XCloseDisplay(_display);
+			Dispose();
 			throw new Exception("xserver doesn't support shm");
 		}
 	}
 
-	public unsafe Stream Capture(Vector2<ushort> resolution, Game? game)
+	public Image Capture(Vector2<ushort> resolution, Game? game)
 	{
 		if (_memorySegment?.Resolution != resolution)
 		{
 			_memorySegment?.Dispose();
 			_memorySegment = new SharedImageMemorySegment<Bgra32>(_display, resolution);
 		}
 		_memorySegment.FetchData(_screen, new Vector2<ushort>());
-		MemoryStream stream = new();
-		ShmImage image = new();
-		Image.LoadPixelData(_memorySegment.Data, resolution.X, resolution.Y).Save(stream, Encoder);
-		XLibShm.DestroyImage(_display, &image);
-		stream.Position = 0;
-		return stream;
+		return Image.LoadPixelData(_memorySegment.Data, resolution.X, resolution.Y);
 	}
 
 	private readonly nint _display;

SightKeeper.Application.Windows/WindowsScreenCapture.cs

@@ -1,8 +1,5 @@
-using System.Drawing;
-using System.Drawing.Imaging;
-using Serilog.Events;
-using SerilogTimings;
-using SightKeeper.Domain.Model;
+using SightKeeper.Domain.Model;
+using Image = SixLabors.ImageSharp.Image;
 
 namespace SightKeeper.Application.Windows;
 
@@ -13,20 +10,16 @@ public WindowsScreenCapture(ScreenBoundsProvider screenBoundsProvider)
 		_screenBoundsProvider = screenBoundsProvider;
 	}
 
-	public Stream Capture(Vector2<ushort> resolution, Game? game)
+	public Image Capture(Vector2<ushort> resolution, Game? game)
 	{
-		var screenCenter = _screenBoundsProvider.MainScreenCenter;
-		var operation = Operation.At(LogEventLevel.Verbose).Begin("Screen capturing");
+		throw new NotImplementedException();
+		/*var screenCenter = _screenBoundsProvider.MainScreenCenter;
 		using Bitmap windowsBitmap = new(resolution.X, resolution.Y);
 		using var graphics = Graphics.FromImage(windowsBitmap);
 		var halfResolution = resolution / 2;
 		Point position = new(screenCenter.X - halfResolution.X, screenCenter.Y - halfResolution.Y);
 		Size size = new(resolution.X, resolution.Y);
-		graphics.CopyFromScreen(position, Point.Empty, size);
-		MemoryStream stream = new();
-		windowsBitmap.Save(stream, ImageFormat.Bmp);
-		operation.Complete();
-		return stream;
+		graphics.CopyFromScreen(position, Point.Empty, size);*/
 	}
 
 	private readonly ScreenBoundsProvider _screenBoundsProvider;

SightKeeper.Application/ScreenCapture.cs

@@ -1,8 +1,9 @@
 using SightKeeper.Domain.Model;
+using SixLabors.ImageSharp;
 
 namespace SightKeeper.Application;
 
 public interface ScreenCapture
 {
-	Stream Capture(Vector2<ushort> resolution, Game? game);
+	Image Capture(Vector2<ushort> resolution, Game? game);
 }

SightKeeper.Application/SightKeeper.Application.csproj

@@ -18,7 +18,7 @@
         <PackageReference Include="SixLabors.ImageSharp" Version="3.1.5" />
         <PackageReference Include="System.Reactive" Version="6.0.1" />
         <PackageReference Include="YamlDotNet" Version="16.0.0" />
-        <PackageReference Include="YoloV8.Gpu" Version="4.1.7" />
+        <PackageReference Include="YoloV8.Gpu" Version="5.0.1" />
     </ItemGroup>
 
     <ItemGroup>

SightKeeper.Avalonia/Program.cs

@@ -1,12 +1,8 @@
 using System;
-using System.IO;
 using System.Threading.Tasks;
 using Avalonia;
 using Avalonia.Threading;
 using Serilog;
-using SightKeeper.Application.Linux;
-using SightKeeper.Domain.Model;
-using SixLabors.ImageSharp;
 
 namespace SightKeeper.Avalonia;
 
@@ -18,26 +14,6 @@ internal static class Program
 	[STAThread]
 	public static void Main(string[] args)
 	{
-		X11ScreenCapture screenCapture = new();
-		using var initial = screenCapture.Capture(new Vector2<ushort>(640, 640), null);
-		if (File.Exists("Test.png"))
-			File.Delete("Test.png");
-		Image.Load(initial).Save("Test.png");
-		// warmup
-		for (int i = 0; i < 200; i++)
-		{
-			using var stream = screenCapture.Capture(new Vector2<ushort>(640, 640), null);
-		}
-		DateTime start = DateTime.UtcNow;
-		const int samples = 1000;
-		for (int i = 0; i < samples; i++)
-		{
-			using var stream = screenCapture.Capture(new Vector2<ushort>(640, 640), null);
-		}
-		var end = DateTime.UtcNow - start;
-		Console.WriteLine($"Elapsed {end.TotalMilliseconds / samples}ms per capture");
-		Console.ReadKey(true);
-		return;
 		SetupLogger();
 		AppBuilder? appBuilder = null;
 		try