Create ycbcr quad sampler pipeline, convert ffmpeg frame to nv12

This commit is contained in:
mittorn 2024-10-28 23:44:11 +03:00
parent 61b25038b1
commit 952dc916f1
4 changed files with 101 additions and 32 deletions

14
quad.frag Normal file
View file

@ -0,0 +1,14 @@
#version 450
layout (location = 0) in vec2 uv;
layout (location = 0) out vec4 outFragColor;
layout (binding = 0) uniform sampler2D ycbcrSampler;
void main()
{
vec3 yuv = texture(ycbcrSampler,uv).xyz;
float y = yuv.y;
float u = yuv.z;// - 0.5;
float v = yuv.x;// - 0.5;
outFragColor = vec4(y + 1.403*v, y - 0.344*u - 0.714*v, y + 1.770*u, 1);
}

27
quad.vert Normal file
View file

@ -0,0 +1,27 @@
#version 450
layout (location = 0) out vec2 uv;
const vec2 src_pos[4] = {
vec2(-1,-1),
vec2(-1,1),
vec2(1,1),
vec2(1,-1)
};
const vec2 src_uv[4] = {
vec2(0,0),
vec2(0,1),
vec2(1,1),
vec2(1,0)
};
out gl_PerVertex
{
vec4 gl_Position;
};
void main()
{
gl_Position = vec4(src_pos[gl_VertexIndex],0,1);
uv = src_uv[gl_VertexIndex];
}

View file

@ -1,4 +1,4 @@
#include "vulkan_pipeline_utl.h"
#include "vulkan_pipeline_utl.h"
#include "vulkan_utl.h"
#include "vulkan_framebuffer_utl.h"
#include "vulkan_texture_utl.h"
@ -23,7 +23,7 @@ extern "C" {
#include <GLFW/glfw3native.h>
// todo: REMOVE THIS. Why "simpliest triangle" demos even use this????
// It does not "HELP" drawing 2D triangle!!! It is not useful in engines using own matrix transform code!
#include <glm/gtc/matrix_transform.hpp>
//#include <glm/gtc/matrix_transform.hpp>
#define MAX_SWAPCHAIN_IMAGES 32
// workaround for ffmpeg stable api nonsence with incompatible pointers between versions
struct PointerWrap
@ -127,10 +127,16 @@ void CreateSoftwareImage(VkInstance inst, VulkanDevice &dev, DecoderImage &image
$(subresourceRange) = SubresourceRange()),
vkCreateImageView(dev.device, &ref, NULL, &image.image_view));
vkMapMemory(dev.device, image.image_memory, 0, image_size, 0, (void**)&image.pMappedData);
VkCommandBuffer cbuf = dev.CreateCommandBuffer();
VulkanTexture::SetImageLayout(cbuf, image.image, VK_IMAGE_LAYOUT_PREINITIALIZED, VK_IMAGE_LAYOUT_GENERAL, { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 });
dev.FlushCommandBuffer(cbuf,dev.defautQueue);
}
#define MAX_DECODER_FRAMES 4
static struct FFmpegDecoder
{
DecoderImage images[4];
DecoderImage images[MAX_DECODER_FRAMES];
// todo: align thread-shared data to unshare with immutable
volatile int decode_index, present_index;
pthread_t thread = 0;
@ -150,6 +156,11 @@ static int WaitActiveFrame()
return ret;
}
static enum AVPixelFormat my_get_format(struct AVCodecContext *s, const enum AVPixelFormat * fmt)
{
return AV_PIX_FMT_YUV420P;
}
static void *DecoderThread(void*)
{
AVFormatContext *input_ctx = NULL;
@ -161,6 +172,7 @@ static void *DecoderThread(void*)
avformat_find_stream_info(input_ctx, NULL);
video = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, PointerWrap(&decoder), 0);
decoder_ctx = avcodec_alloc_context3(decoder);
decoder_ctx->get_format = my_get_format;
avcodec_open2(decoder_ctx, decoder, NULL);
packet = av_packet_alloc();
AVFrame *frame = av_frame_alloc();
@ -180,8 +192,16 @@ static void *DecoderThread(void*)
idx++;
DecoderImage &img = gFF.images[idx & 3];
// todo: wait fence?
memcpy(img.pMappedData + img.memory_offset_plane0 , frame->data[0], frame->linesize[0] + frame->height);
memcpy(img.pMappedData + img.memory_offset_plane1 , frame->data[1], frame->linesize[1] + frame->height/2);
memcpy(img.pMappedData + img.memory_offset_plane0 , frame->data[0], frame->linesize[0] * frame->height);
// ffmpeg cannot NV12????
for(int i = 0; i < frame->height / 2; i++)
for(int j = 0; j < frame->linesize[1]; j++)
{
*(img.pMappedData + img.memory_offset_plane1 + frame->linesize[1]*2*i +j * 2) = *(frame->data[1] + frame->linesize[1]* i + j);
*(img.pMappedData + img.memory_offset_plane1 + frame->linesize[1]*2*i +j * 2+1) = *(frame->data[2] + frame->linesize[2]* i + j);
}
//memcpy(img.pMappedData + img.memory_offset_plane1 , frame->data[1], frame->linesize[1] * frame->height/2);
//assert(!frame->data[2]);
pthread_mutex_lock(&gFF.mutex);
gFF.decode_index = idx;
pthread_cond_signal(&gFF.cond);
@ -198,51 +218,54 @@ static void *DecoderThread(void*)
static void SetupFFMpeg(VkInstance inst, VulkanDevice &dev)
{
for(int i = 0; i < 4; i++)
for(int i = 0; i < MAX_DECODER_FRAMES; i++)
CreateSoftwareImage(inst, dev, gFF.images[i]);
pthread_mutex_init(&gFF.mutex, NULL);
pthread_cond_init(&gFF.cond, NULL);
pthread_create(&gFF.thread, NULL, &DecoderThread, NULL);
}
#define RECONSTRUCTION_TARGET_FRAMES 4
struct GraphicsApplicationPipeline: BaseVulkanPipeline
{
void Init(VkDevice dev, VkRenderPass renderPass)
void Init(VkDevice dev, VkRenderPass renderPass, VkSampler *pImmutableSamplers)
{
device = dev;
CreateDescriptorSetLayout(
BasicBinding(0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,1,VK_SHADER_STAGE_VERTEX_BIT)
//BasicBinding(0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,1,VK_SHADER_STAGE_VERTEX_BIT)
BasicBinding(0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,1,VK_SHADER_STAGE_FRAGMENT_BIT, pImmutableSamplers)
);
CreatePool(1,
BasicPoolSize(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1)
//BasicPoolSize(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1)
BasicPoolSize(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1)
);
VkShaderModule vp, fp;
CreateGraphicsPipeline(renderPass,
Stages(
ShaderFromFile(vp, "triangle.vert.spv", VK_SHADER_STAGE_VERTEX_BIT),
ShaderFromFile(fp, "triangle.frag.spv", VK_SHADER_STAGE_FRAGMENT_BIT)),
VertexBindings(
VertexBinding(0, sizeof(Vertex))
),
VertexAttributes(
VertAttrib(0, 0, offsetof(Vertex,position)),
VertAttrib(1, 0, offsetof(Vertex,color))
),
ShaderFromFile(vp, "quad.vert.spv", VK_SHADER_STAGE_VERTEX_BIT),
ShaderFromFile(fp, "quad.frag.spv", VK_SHADER_STAGE_FRAGMENT_BIT)),
VertexBindings(),
VertexAttributes(),
DynamicStates(
VK_DYNAMIC_STATE_VIEWPORT,
VK_DYNAMIC_STATE_SCISSOR
),
DepthStencil(true, true)
DepthStencil(true, true),
AssemblyTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN)
//RasterMode(VK_FRONT_FACE_CLOCKWISE)
);
vkDestroyShaderModule(device, vp, NULL);
vkDestroyShaderModule(device, fp, NULL);
}
void UpdateDescriptors(VkDescriptorSet dstSet, const VkDescriptorBufferInfo &buffer)
void UpdateDescriptors(VkDescriptorSet dstSet, VkImageView imageView, VkSampler sampler)
{
WriteDescriptors(
BufferWrite(dstSet, 0, buffer)
//BufferWrite(dstSet, 0, buffer)
ImageWrite(dstSet, 0, ImageDescriptor(imageView, VK_IMAGE_LAYOUT_GENERAL, sampler), VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
);
}
};
@ -257,12 +280,13 @@ struct GraphicsApplication
VkImage swapchainImages[MAX_SWAPCHAIN_IMAGES];
unsigned int numSwapchainImages;
VulkanFramebuffer swapchainFbs[MAX_SWAPCHAIN_IMAGES];
VkCommandBuffer commandBuffers[MAX_SWAPCHAIN_IMAGES];
VkCommandBuffer commandBuffers[MAX_SWAPCHAIN_IMAGES * RECONSTRUCTION_TARGET_FRAMES];
VkCommandBuffer reconstructionCommandBuffers[MAX_DECODER_FRAMES * RECONSTRUCTION_TARGET_FRAMES];
VkFence chainFences[MAX_SWAPCHAIN_IMAGES];
VkSemaphore swapchainRenderSemaphore[MAX_SWAPCHAIN_IMAGES], swapchainPresentSemaphore[MAX_SWAPCHAIN_IMAGES];
struct UBO
{
glm::mat4 transformationMatrix;
//glm::mat4 transformationMatrix;
};
GraphicsApplicationPipeline graphicsPipeline;
@ -364,6 +388,7 @@ struct GraphicsApplication
static unsigned int frameNum;
// Rotate based on time
long long millis = frameNum++;
#if 0
float angle = (millis % 400) / 400.0f * glm::radians(360.f);
glm::mat4 modelMatrix = glm::identity<glm::mat4>();
@ -377,6 +402,7 @@ struct GraphicsApplication
// Set up projection
auto projMatrix = glm::perspective(glm::radians(70.f), 800.0f / 600.0f, 0.1f, 10.0f);
((UBO*)uboBuf.mapped)->transformationMatrix = projMatrix * viewMatrix * modelMatrix;
#endif
}
void waitFence(int chidx)
@ -399,9 +425,10 @@ struct GraphicsApplication
CreateWindow("demo", width, height, false);
CreateSwapchain(width,height);
dev.CreateAndMap(uboBuf, sizeof(UBO));
graphicsPipeline.Init(dev.device, swapchainFbs[0].render_pass);
SetupFFMpeg(context.instance, dev);
graphicsPipeline.Init(dev.device, swapchainFbs[0].render_pass, &gFF.images[0].ycbcr_sampler);
descriptorSet = graphicsPipeline.AllocateSingleDescriptorSet();
graphicsPipeline.UpdateDescriptors(descriptorSet, uboBuf.descriptor);
graphicsPipeline.UpdateDescriptors(descriptorSet, gFF.images[0].image_view, gFF.images[0].ycbcr_sampler);
updateUniformData();
Vertex vertices[] = {
@ -424,17 +451,18 @@ struct GraphicsApplication
vkCmdBindDescriptorSets(commandBuffers[i],VK_PIPELINE_BIND_POINT_GRAPHICS,graphicsPipeline.pipelineLayout, 0, 1, &descriptorSet, 0, nullptr);
vkCmdBindPipeline(commandBuffers[i], VK_PIPELINE_BIND_POINT_GRAPHICS,graphicsPipeline.pipeline);
VkDeviceSize offset = 0;
vkCmdBindVertexBuffers(commandBuffers[i],0, 1, &stagingVert.buffer, &offset);
vkCmdBindIndexBuffer(commandBuffers[i], stagingInd.buffer, 0, VK_INDEX_TYPE_UINT32);
vkCmdDrawIndexed(commandBuffers[i], 3, 1, 0, 0, 0);
// VkDeviceSize offset = 0;
//vkCmdBindVertexBuffers(commandBuffers[i],0, 1, &stagingVert.buffer, &offset);
//vkCmdBindIndexBuffer(commandBuffers[i], stagingInd.buffer, 0, VK_INDEX_TYPE_UINT32);
//vkCmdDrawIndexed(commandBuffers[i], 3, 1, 0, 0, 0);
vkCmdDraw(commandBuffers[i], 4, 1, 0, 0);
vkCmdEndRenderPass(commandBuffers[i]);
VulkanTexture::SetImageLayout(commandBuffers[i], swapchainImages[i], VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 });
vkEndCommandBuffer(commandBuffers[i]);
VK_CHECK_RESULT(CallWith(FenceInfo(VK_FENCE_CREATE_SIGNALED_BIT),
vkCreateFence(dev.device, &ref, NULL, &chainFences[i])));
}
SetupFFMpeg(context.instance, dev);
uint32_t idx = 0;
int sem_idx = 0;
while(true)

View file

@ -311,13 +311,13 @@ struct BaseVulkanPipeline
return {sampler, imageView, imageLayout};
}
// todo: different types
VkWriteDescriptorSet ImageWrite(VkDescriptorSet dstSet, uint32_t binding, const VkDescriptorImageInfo &info)
VkWriteDescriptorSet ImageWrite(VkDescriptorSet dstSet, uint32_t binding, const VkDescriptorImageInfo &info, VkDescriptorType tp = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE)
{
VkWriteDescriptorSet wr = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
wr.dstSet = dstSet;
wr.dstBinding = binding;
wr.descriptorCount = 1;
wr.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
wr.descriptorType = tp;
wr.pImageInfo = &info;
return wr;
}