Skip to content

Commit

Permalink
calculate stride for screen capture.
Browse files Browse the repository at this point in the history
  • Loading branch information
@sblantipodi
sblantipodi committed Dec 18, 2023
1 parent c9f0d18 commit cd04656
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 35 deletions.
12 changes: 6 additions & 6 deletions src/main/java/org/dpsoftware/config/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,11 +480,6 @@ public class Constants {
public static final String INTERNAL_SCALING_Y = "INTERNAL_SCALING_Y";
public static final int RESAMPLING_FACTOR = 4;
public static final String EMIT_SIGNALS = "emit-signals";
public static final String GSTREAMER_PIPELINE_DDUPL_SM = "video/x-raw(memory:SystemMemory),width=INTERNAL_SCALING_X,height=INTERNAL_SCALING_Y,sync=false,";
public static final String GSTREAMER_PIPELINE_DDUPL = "video/x-raw(memory:D3D11Memory),width=INTERNAL_SCALING_X,height=INTERNAL_SCALING_Y,sync=false,";
public static final String GSTREAMER_PIPELINE = "video/x-raw,width=INTERNAL_SCALING_X,height=INTERNAL_SCALING_Y,sync=false,";
public static final String BYTE_ORDER_BGR = "format=BGRx";
public static final String BYTE_ORDER_RGB = "format=xRGB";
public static final String WIDTH = "width";
public static final String HEIGHT = "height";
public static final String GSTREAMER_PATH = "/gstreamer/1.0/mingw_x86_64/bin";
Expand All @@ -497,13 +492,18 @@ public class Constants {
// ./gst-launch-1.0 d3d11screencapturesrc ! d3d11convert ! "video/x-raw(memory:D3D11Memory),width=800,height=600,sync=false" ! autovideosink
// ./gst-launch-1.0 d3d11screencapturesrc ! d3d11convert ! d3d11download ! "video/x-raw(memory:SystemMemory),width=480,height=270,sync=false" ! autovideosink
// ./gst-launch-1.0 ximagesrc startx=0 endx=3839 starty=0 endy=2159 use-damage=0 ! videoscale ! videoconvert ! autovideosink
public static final String GSTREAMER_PIPELINE_WINDOWS_HARDWARE_HANDLE_SM = "d3d11screencapturesrc monitor-handle={0} ! d3d11convert ! d3d11download";
// public static final String GSTREAMER_PIPELINE_WINDOWS_HARDWARE_HANDLE_CPU_SCALING = "d3d11screencapturesrc monitor-handle={0} ! d3d11convert ! d3d11download";
public static final String GSTREAMER_PIPELINE_WINDOWS_HARDWARE_HANDLE = "d3d11screencapturesrc monitor-handle={0} ! d3d11convert";
public static final String GSTREAMER_PIPELINE_XIMAGESRC = "ximagesrc startx={0} endx={1} starty={2} endy={3} use-damage=0 ! videoscale ! videoconvert";
public static final String GSTREAMER_PIPELINE_XIMAGESRC_CUDA = "ximagesrc startx={0} endx={1} starty={2} endy={3} use-damage=0 ! cudaupload ! cudascale ! cudaconvert ! cudadownload";
public static final String GSTREAMER_PIPELINE_PIPEWIREXDG = "pipewiresrc fd={1} path={2} ! videorate ! videoscale ! videoconvert";
public static final String GSTREAMER_PIPELINE_PIPEWIREXDG_CUDA = "pipewiresrc fd={1} path={2} ! videorate ! cudaupload ! cudascale ! cudaconvert ! cudadownload";
public static final String GSTREAMER_PIPELINE_MAC = "avfvideosrc capture-screen=true ! videoscale ! videoconvert";
// public static final String GSTREAMER_PIPELINE_DDUPL_CPU_SCALING = "video/x-raw(memory:SystemMemory),width=INTERNAL_SCALING_X,height=INTERNAL_SCALING_Y,sync=false,";
public static final String GSTREAMER_PIPELINE_DDUPL = "video/x-raw(memory:D3D11Memory),width=INTERNAL_SCALING_X,height=INTERNAL_SCALING_Y,sync=false,";
public static final String GSTREAMER_PIPELINE = "video/x-raw,width=INTERNAL_SCALING_X,height=INTERNAL_SCALING_Y,sync=false,";
public static final String BYTE_ORDER_BGR = "format=BGRx";
public static final String BYTE_ORDER_RGB = "format=xRGB";
public static final String FRAMERATE_PLACEHOLDER = "framerate=FRAMERATE_PLACEHOLDER/1,";
public static final String FPS_PLACEHOLDER = "FRAMERATE_PLACEHOLDER";
public static final String GSTREAMER_SCREENSHOT = "gstreamer_screenshot.bmp";
Expand Down
48 changes: 28 additions & 20 deletions src/main/java/org/dpsoftware/grabber/GStreamerGrabber.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,10 @@ public GStreamerGrabber(AppSink appsink) {
videosink.connect(listener);
String gstreamerPipeline;
if (MainSingleton.getInstance().config.getCaptureMethod().equals(Configuration.CaptureMethod.DDUPL.name())) {
// Scale image inside the GPU by RESAMPLING_FACTOR, Constants.GSTREAMER_MEMORY_DIVIDER tells if resolution is compatible with D3D11Memory with no padding.
if (!GrabberSingleton.getInstance().isFallbackPipeline()) {
gstreamerPipeline = Constants.GSTREAMER_PIPELINE_DDUPL
.replace(Constants.INTERNAL_SCALING_X, String.valueOf(MainSingleton.getInstance().config.getScreenResX() / Constants.RESAMPLING_FACTOR))
.replace(Constants.INTERNAL_SCALING_Y, String.valueOf(MainSingleton.getInstance().config.getScreenResY() / Constants.RESAMPLING_FACTOR));
} else {
gstreamerPipeline = Constants.GSTREAMER_PIPELINE_DDUPL_SM
.replace(Constants.INTERNAL_SCALING_X, String.valueOf(MainSingleton.getInstance().config.getScreenResX() / Constants.RESAMPLING_FACTOR))
.replace(Constants.INTERNAL_SCALING_Y, String.valueOf(MainSingleton.getInstance().config.getScreenResY() / Constants.RESAMPLING_FACTOR));
}
// Scale image inside the GPU by RESAMPLING_FACTOR
gstreamerPipeline = Constants.GSTREAMER_PIPELINE_DDUPL
.replace(Constants.INTERNAL_SCALING_X, String.valueOf(MainSingleton.getInstance().config.getScreenResX() / Constants.RESAMPLING_FACTOR))
.replace(Constants.INTERNAL_SCALING_Y, String.valueOf(MainSingleton.getInstance().config.getScreenResY() / Constants.RESAMPLING_FACTOR));
} else {
gstreamerPipeline = Constants.GSTREAMER_PIPELINE
.replace(Constants.INTERNAL_SCALING_X, String.valueOf(MainSingleton.getInstance().config.getScreenResX() / Constants.RESAMPLING_FACTOR))
Expand Down Expand Up @@ -167,6 +161,26 @@ private void intBufferRgbToImage(IntBuffer rgbBuffer) {
}
}

/**
* Return the stride
*
* @param width captured image width (includes rescaling)
* @param height captured image height (includes rescaling)
* @param rgbBuffer captured image IntBuffer
* @return width that contains stride for some resolutions that needs it like: 3440x1440 on NVIDIA or 1920x1080 on AMD
*/
private int getWidthPlusStride(int width, int height, IntBuffer rgbBuffer) {
int widthPlusStride = width;
final int exectedCapacityWithoutStride = width * height;
if ((rgbBuffer.capacity()) != exectedCapacityWithoutStride) {
int capacity = rgbBuffer.capacity();
int difference = capacity - exectedCapacityWithoutStride;
int stride = difference / height;
widthPlusStride = width + stride;
}
return widthPlusStride;
}

/**
* Listener callback triggered every frame
*/
Expand All @@ -178,13 +192,6 @@ public void rgbFrame(int width, int height, IntBuffer rgbBuffer) {
if (!bufferLock.tryLock()) {
return;
}
int intBufferSize = (width * height) - 1;
if (((rgbBuffer.capacity() - 1) != intBufferSize) && !GrabberSingleton.getInstance().isFallbackPipeline()) {
log.debug("Received buffer is different from the expected, using fallback pipeline.");
GrabberSingleton.getInstance().setFallbackPipeline(true);
PipelineManager.restartCapture(() -> {
}, true);
}
// CHECK_ASPECT_RATIO is true 10 times per second, if true and black bars auto detection is on, auto detect black bars
if (MainSingleton.getInstance().config.isAutoDetectBlackBars()) {
if (GrabberSingleton.getInstance().CHECK_ASPECT_RATIO) {
Expand All @@ -197,6 +204,7 @@ public void rgbFrame(int width, int height, IntBuffer rgbBuffer) {
if (MainSingleton.getInstance().config.getRuntimeLogLevel().equals(Level.TRACE.levelStr)) {
intBufferRgbToImage(rgbBuffer);
}
int widthPlusStride = getWidthPlusStride(width, height, rgbBuffer);
// We need an ordered collection so no parallelStream here
ledMatrix.forEach((key, value) -> {
int r = 0, g = 0, b = 0;
Expand All @@ -213,9 +221,9 @@ public void rgbFrame(int width, int height, IntBuffer rgbBuffer) {
for (int x = 0; x < pixelInUseX; x++) {
int offsetX = (xCoordinate + (skipPixel * x));
int offsetY = (yCoordinate + (skipPixel * y));
int bufferOffset = (Math.min(offsetX, width))
+ ((offsetY < height) ? (offsetY * width) : (height * width));
int rgb = rgbBuffer.get(Math.min(intBufferSize, bufferOffset));
int bufferOffset = (Math.min(offsetX, widthPlusStride))
+ ((offsetY < height) ? (offsetY * widthPlusStride) : (height * widthPlusStride));
int rgb = rgbBuffer.get(Math.min(rgbBuffer.capacity() - 1, bufferOffset));
r += rgb >> 16 & 0xFF;
g += rgb >> 8 & 0xFF;
b += rgb & 0xFF;
Expand Down
7 changes: 1 addition & 6 deletions src/main/java/org/dpsoftware/grabber/GrabberManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,7 @@ public void launchAdvancedGrabber(ImageProcessor imageProcessor) {
if (NativeExecutor.isWindows()) {
DisplayManager displayManager = new DisplayManager();
String monitorNativePeer = String.valueOf(displayManager.getDisplayInfo(MainSingleton.getInstance().config.getMonitorNumber()).getNativePeer());
// Constants.GSTREAMER_MEMORY_DIVIDER tells if resolution is compatible with D3D11Memory with no padding.
if (!GrabberSingleton.getInstance().isFallbackPipeline()) {
bin = Gst.parseBinFromDescription(Constants.GSTREAMER_PIPELINE_WINDOWS_HARDWARE_HANDLE.replace("{0}", monitorNativePeer), true);
} else {
bin = Gst.parseBinFromDescription(Constants.GSTREAMER_PIPELINE_WINDOWS_HARDWARE_HANDLE_SM.replace("{0}", monitorNativePeer), true);
}
bin = Gst.parseBinFromDescription(Constants.GSTREAMER_PIPELINE_WINDOWS_HARDWARE_HANDLE.replace("{0}", monitorNativePeer), true);
} else if (NativeExecutor.isLinux()) {
bin = Gst.parseBinFromDescription(finalLinuxParams, true);
} else {
Expand Down
3 changes: 0 additions & 3 deletions src/main/java/org/dpsoftware/grabber/GrabberSingleton.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,6 @@ public class GrabberSingleton {
public Rectangle rect;
// GStreamer Rendering pipeline
public Pipeline pipe;
// There is a known issue that prevents to correcly scale the captured image with some resolutions/GPUs.
// for example 3440x1440 on NVIDIA, 1920x1080 on AMD. Scale the image on the CPU if this is the case.
public boolean fallbackPipeline;
float maxPeak, maxRms = 0;
float maxPeakLeft, maxRmsLeft = 0;
float maxPeakRight, maxRmsRight = 0;
Expand Down

0 comments on commit cd04656

Please sign in to comment.