Send rendered data to encoder (broken, does not produce PPS/SPS)

vaapi-recorder.cpp

@@ -722,7 +722,7 @@ encoder_render_picture(struct vaapi_recorder *r, VASurfaceID input,
 	if (status != VA_STATUS_SUCCESS)
 		return status;
 
-	return vaSyncSurface(r->va_dpy, input);
+	return 0;//vaSyncSurface(r->va_dpy, input);
 }
 
 static VABufferID
@@ -981,6 +981,70 @@ err_free:
 	return NULL;
 }
 
+static VASurfaceID gInputRGBA;
+
+static VAStatus
+create_surface_from_fd(struct vaapi_recorder *r, int prime_fd,
+			   int stride, VASurfaceID *surface);
+struct vaapi_recorder *
+vaapi_recorder_create2(int drm_fd, int width, int height, const char *filename, int dmabuf_fd, int dmabuf_stride)
+{
+	struct vaapi_recorder *r;
+	VAStatus status;
+	int major, minor;
+	int flags;
+
+	r = (vaapi_recorder*)calloc(sizeof *r,1);
+	if (r == NULL)
+		return NULL;
+
+	r->width = width;
+	r->height = height;
+	r->drm_fd = drm_fd;
+
+	flags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC;
+	r->output_fd = open(filename, flags, 0644);
+	if (r->output_fd < 0)
+		goto err_thread;
+
+	r->va_dpy = vaGetDisplayDRM(drm_fd);
+	if (!r->va_dpy) {
+		printf("failed to create VA display\n");
+		goto err_fd;
+	}
+
+	status = vaInitialize(r->va_dpy, &major, &minor);
+	if (status != VA_STATUS_SUCCESS) {
+		printf("vaapi: failed to initialize display\n");
+		goto err_fd;
+	}
+
+	if (setup_vpp(r) < 0) {
+		printf("vaapi: failed to initialize VPP pipeline\n");
+		goto err_va_dpy;
+	}
+
+	if (setup_encoder(r) < 0) {
+		goto err_vpp;
+	}
+	create_surface_from_fd(r, dmabuf_fd, dmabuf_stride, &gInputRGBA);
+
+	return r;
+
+err_vpp:
+	vpp_destroy(r);
+err_va_dpy:
+	vaTerminate(r->va_dpy);
+err_fd:
+	close(r->output_fd);
+err_thread:
+	destroy_worker_thread(r);
+err_free:
+	free(r);
+
+	return NULL;
+}
+
 void
 vaapi_recorder_destroy(struct vaapi_recorder *r)
 {
@@ -1075,6 +1139,7 @@ convert_rgb_to_yuv(struct vaapi_recorder *r, VASurfaceID rgb_surface)
 	return status;
 }
 
+
 static void
 recorder_frame(struct vaapi_recorder *r)
 {
@@ -1103,6 +1168,22 @@ recorder_frame(struct vaapi_recorder *r)
 	vaDestroySurfaces(r->va_dpy, &rgb_surface, 1);
 }
 
+void
+recorder_frame2(struct vaapi_recorder *r)
+{
+	VAStatus status;
+
+	status = convert_rgb_to_yuv(r, gInputRGBA);
+	if (status != VA_STATUS_SUCCESS) {
+		printf("[libva recorder] "
+			   "color space conversion failed\n");
+		return;
+	}
+
+	encoder_encode(r, r->vpp.output);
+}
+
+
 static void *
 worker_thread_function(void *data)
 {

vaapi-recorder.h

@@ -30,9 +30,15 @@ struct vaapi_recorder;
 
 struct vaapi_recorder *
 vaapi_recorder_create(int drm_fd, int width, int height, const char *filename);
+struct vaapi_recorder *
+vaapi_recorder_create2(int drm_fd, int width, int height, const char *filename, int dmabuf_fd, int dmabuf_stride);
+
 void
 vaapi_recorder_destroy(struct vaapi_recorder *r);
 int
 vaapi_recorder_frame(struct vaapi_recorder *r, int fd, int stride);
 
+void
+recorder_frame2(struct vaapi_recorder *r);
+
 #endif /* _VAAPI_RECORDER_H_ */

vkcompute.cpp

@@ -35,6 +35,7 @@
 #include <sys/ioctl.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include "vaapi-recorder.h"
 
 struct DrmHelper
 {
@@ -246,6 +247,7 @@ public:
 		// Buffer size of the storage buffer that will contain the rendered mandelbrot set.
 		bufferSize = sizeof(Pixel) * WIDTH * HEIGHT;
 
+
 		// Initialize vulkan:
 		createInstance();
 		findPhysicalDevice();
@@ -253,6 +255,8 @@ public:
 		createBuffer();
 		vkMapMemory(device, bufferMemory, 0, sizeof(UBO), 0, (void**)&pMappedBuffer);
 		createImageExportableDmabuf();
+		int drm_fd = drm_fd = open("/dev/dri/renderD128", O_RDWR);
+		auto *r = vaapi_recorder_create2(drm_fd, WIDTH, 2300, "out.264", prime_fd, WIDTH * 4);
 		createDescriptorSetLayout();
 		//createDescriptorSet();
 		createComputePipeline();
@@ -272,6 +276,7 @@ public:
 		// Finally, run the recorded command buffer.
 			runCommandBuffer();
 			pMappedBuffer->frameNum = frameNum;
+			recorder_frame2(r);
 		}
 
 		vkDestroyFence(device, fence, NULL);
@@ -811,6 +816,7 @@ public:
 		view.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 };
 		VK_CHECK_RESULT(vkCreateImageView(device, &view, nullptr, &imageView));
 	}
+	int prime_fd;
 
 	// create and import dmabuf as opaque fd, allows any tiling
 	// create and import dmabuf
@@ -894,9 +900,8 @@ public:
 		VkMemoryGetFdInfoKHR getFdInfo = { VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR};
 		getFdInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
 		getFdInfo.memory = imageMemory;
-		int fd = -1;
 		PFN_vkGetMemoryFdKHR pfnvkGetMemoryFdKHR = (PFN_vkGetMemoryFdKHR)vkGetInstanceProcAddr(instance, "vkGetMemoryFdKHR");
-		pfnvkGetMemoryFdKHR(device, &getFdInfo, &fd);
+		pfnvkGetMemoryFdKHR(device, &getFdInfo, &prime_fd);
 		VkImageDrmFormatModifierPropertiesEXT imageModifiers = {VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT};
 		PFN_vkGetImageDrmFormatModifierPropertiesEXT pfnvkGetImageDrmFormatModifierPropertiesEXT =(PFN_vkGetImageDrmFormatModifierPropertiesEXT)vkGetInstanceProcAddr(instance, "vkGetImageDrmFormatModifierPropertiesEXT");
 		pfnvkGetImageDrmFormatModifierPropertiesEXT(device, image, &imageModifiers);
@@ -1166,8 +1171,12 @@ public:
 		submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
 		submitInfo.commandBufferCount = 1; // submit a single command buffer
 		submitInfo.pCommandBuffers = &commandBuffer; // the command buffer to submit.
-		vkResetFences(device, 1, &fence);
 
+	//	static bool b;
+		//if(b)
+			//VK_CHECK_RESULT(vkWaitForFences(device, 1, &fence, VK_TRUE, 100000000000));
+		//b = 1;
+		vkResetFences(device, 1, &fence);
 
 		/*
 		We submit the command buffer on the queue, at the same time giving a fence.