pkg/54409: ffmpeg4 cannot capture video via X11grab

[aran%100acres.us@localhost]

>Number:         54409
>Category:       pkg
>Synopsis:       ffmpeg4 cannot capture video via X11grab
>Confidential:   no
>Severity:       non-critical
>Priority:       medium
>Responsible:    pkg-manager
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Thu Jul 25 15:10:00 +0000 2019
>Originator:     Aran Clauson
>Release:        2019Q2
>Organization:
Personal
>Environment:
NetBSD owl 8.99.47 NetBSD 8.99.47 (GENERIC) #0: Mon Jun 24 09:49:56 PDT 2019  aran@owl:/home/NetBSD/obj/sys/arch/amd64/compile/GENERIC amd64

>Description:
ffmpeg cannot capture video from xorg.  For example, the following command produces an error and no usable output.

ffmpeg4 -r 30 -f x11grab -s 800x600 -i :0.0+1080,869 -c:v libx264 -pix_fmt yuv420p -qp 0 -preset ultrafast -t 10 -y out.mov

>How-To-Repeat:
ffmpeg4 -r 30 -f x11grab -s 800x600 -i :0.0+1080,869 -c:v libx264 -pix_fmt yuv420p -qp 0 -preset ultrafast -t 10 -y out.mov

>Fix:
The following patch fixes the problem.  I've sent this to the FFmpeg development team.  I don't know if they will incorporate the patch. 

$NetBSD$

--- libavdevice/xcbgrab.c.orig	2019-07-25 14:58:08.498136541 +0000
+++ libavdevice/xcbgrab.c
@@ -75,6 +75,7 @@ typedef struct XCBGrabContext {
     const char *framerate;
 
     int has_shm;
+    int shmid;
 } XCBGrabContext;
 
 #define FOLLOW_CENTER -1
@@ -221,6 +222,14 @@ static int check_shm(xcb_connection_t *c
     return 0;
 }
 
+static void rm_shmid(AVFormatContext *s) {
+    XCBGrabContext *c = s->priv_data;
+    if(c->shmid != -1) {
+      shmctl(c->shmid, IPC_RMID, 0);
+      c->shmid = -1;
+    }
+}
+
 static int allocate_shm(AVFormatContext *s)
 {
     XCBGrabContext *c = s->priv_data;
@@ -230,7 +239,8 @@ static int allocate_shm(AVFormatContext 
 
     if (c->buffer)
         return 0;
-    id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0777);
+    
+    id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0666);
     if (id == -1) {
         char errbuf[1024];
         int err = AVERROR(errno);
@@ -239,15 +249,20 @@ static int allocate_shm(AVFormatContext 
                size, errbuf);
         return err;
     }
+    
     xcb_shm_attach(c->conn, c->segment, id, 0);
     data = shmat(id, NULL, 0);
-    shmctl(id, IPC_RMID, 0);
-    if ((intptr_t)data == -1 || !data)
-        return AVERROR(errno);
+
+    if ((intptr_t)data == -1 || !data) {
+      shmctl(id, IPC_RMID, 0);
+      return AVERROR(errno);
+    }
     c->buffer = data;
+    c->shmid = id;
     return 0;
 }
 
+
 static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)
 {
     XCBGrabContext *c = s->priv_data;
@@ -268,6 +283,8 @@ static int xcbgrab_frame_shm(AVFormatCon
 
     xcb_flush(c->conn);
 
+    rm_shmid(s);
+
     if (e) {
         av_log(s, AV_LOG_ERROR,
                "Cannot get the image data "