Create ycbcr quad sampler pipeline, convert ffmpeg frame to nv12
This commit is contained in:
parent
61b25038b1
commit
952dc916f1
4 changed files with 101 additions and 32 deletions
14
quad.frag
Normal file
14
quad.frag
Normal file
|
@ -0,0 +1,14 @@
|
|||
#version 450
|
||||
|
||||
layout (location = 0) in vec2 uv;
|
||||
|
||||
layout (location = 0) out vec4 outFragColor;
|
||||
layout (binding = 0) uniform sampler2D ycbcrSampler;
|
||||
void main()
|
||||
{
|
||||
vec3 yuv = texture(ycbcrSampler,uv).xyz;
|
||||
float y = yuv.y;
|
||||
float u = yuv.z;// - 0.5;
|
||||
float v = yuv.x;// - 0.5;
|
||||
outFragColor = vec4(y + 1.403*v, y - 0.344*u - 0.714*v, y + 1.770*u, 1);
|
||||
}
|
27
quad.vert
Normal file
27
quad.vert
Normal file
|
@ -0,0 +1,27 @@
|
|||
#version 450
|
||||
|
||||
layout (location = 0) out vec2 uv;
|
||||
const vec2 src_pos[4] = {
|
||||
vec2(-1,-1),
|
||||
vec2(-1,1),
|
||||
vec2(1,1),
|
||||
vec2(1,-1)
|
||||
};
|
||||
|
||||
const vec2 src_uv[4] = {
|
||||
vec2(0,0),
|
||||
vec2(0,1),
|
||||
vec2(1,1),
|
||||
vec2(1,0)
|
||||
};
|
||||
|
||||
out gl_PerVertex
|
||||
{
|
||||
vec4 gl_Position;
|
||||
};
|
||||
|
||||
void main()
|
||||
{
|
||||
gl_Position = vec4(src_pos[gl_VertexIndex],0,1);
|
||||
uv = src_uv[gl_VertexIndex];
|
||||
}
|
88
vkplayer.cpp
88
vkplayer.cpp
|
@ -1,4 +1,4 @@
|
|||
#include "vulkan_pipeline_utl.h"
|
||||
#include "vulkan_pipeline_utl.h"
|
||||
#include "vulkan_utl.h"
|
||||
#include "vulkan_framebuffer_utl.h"
|
||||
#include "vulkan_texture_utl.h"
|
||||
|
@ -23,7 +23,7 @@ extern "C" {
|
|||
#include <GLFW/glfw3native.h>
|
||||
// todo: REMOVE THIS. Why "simpliest triangle" demos even use this????
|
||||
// It does not "HELP" drawing 2D triangle!!! It is not useful in engines using own matrix transform code!
|
||||
#include <glm/gtc/matrix_transform.hpp>
|
||||
//#include <glm/gtc/matrix_transform.hpp>
|
||||
#define MAX_SWAPCHAIN_IMAGES 32
|
||||
// workaround for ffmpeg stable api nonsence with incompatible pointers between versions
|
||||
struct PointerWrap
|
||||
|
@ -127,10 +127,16 @@ void CreateSoftwareImage(VkInstance inst, VulkanDevice &dev, DecoderImage &image
|
|||
$(subresourceRange) = SubresourceRange()),
|
||||
vkCreateImageView(dev.device, &ref, NULL, &image.image_view));
|
||||
vkMapMemory(dev.device, image.image_memory, 0, image_size, 0, (void**)&image.pMappedData);
|
||||
VkCommandBuffer cbuf = dev.CreateCommandBuffer();
|
||||
VulkanTexture::SetImageLayout(cbuf, image.image, VK_IMAGE_LAYOUT_PREINITIALIZED, VK_IMAGE_LAYOUT_GENERAL, { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 });
|
||||
dev.FlushCommandBuffer(cbuf,dev.defautQueue);
|
||||
}
|
||||
|
||||
#define MAX_DECODER_FRAMES 4
|
||||
|
||||
static struct FFmpegDecoder
|
||||
{
|
||||
DecoderImage images[4];
|
||||
DecoderImage images[MAX_DECODER_FRAMES];
|
||||
// todo: align thread-shared data to unshare with immutable
|
||||
volatile int decode_index, present_index;
|
||||
pthread_t thread = 0;
|
||||
|
@ -150,6 +156,11 @@ static int WaitActiveFrame()
|
|||
return ret;
|
||||
}
|
||||
|
||||
static enum AVPixelFormat my_get_format(struct AVCodecContext *s, const enum AVPixelFormat * fmt)
|
||||
{
|
||||
return AV_PIX_FMT_YUV420P;
|
||||
}
|
||||
|
||||
static void *DecoderThread(void*)
|
||||
{
|
||||
AVFormatContext *input_ctx = NULL;
|
||||
|
@ -161,6 +172,7 @@ static void *DecoderThread(void*)
|
|||
avformat_find_stream_info(input_ctx, NULL);
|
||||
video = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, PointerWrap(&decoder), 0);
|
||||
decoder_ctx = avcodec_alloc_context3(decoder);
|
||||
decoder_ctx->get_format = my_get_format;
|
||||
avcodec_open2(decoder_ctx, decoder, NULL);
|
||||
packet = av_packet_alloc();
|
||||
AVFrame *frame = av_frame_alloc();
|
||||
|
@ -180,8 +192,16 @@ static void *DecoderThread(void*)
|
|||
idx++;
|
||||
DecoderImage &img = gFF.images[idx & 3];
|
||||
// todo: wait fence?
|
||||
memcpy(img.pMappedData + img.memory_offset_plane0 , frame->data[0], frame->linesize[0] + frame->height);
|
||||
memcpy(img.pMappedData + img.memory_offset_plane1 , frame->data[1], frame->linesize[1] + frame->height/2);
|
||||
memcpy(img.pMappedData + img.memory_offset_plane0 , frame->data[0], frame->linesize[0] * frame->height);
|
||||
// ffmpeg cannot NV12????
|
||||
for(int i = 0; i < frame->height / 2; i++)
|
||||
for(int j = 0; j < frame->linesize[1]; j++)
|
||||
{
|
||||
*(img.pMappedData + img.memory_offset_plane1 + frame->linesize[1]*2*i +j * 2) = *(frame->data[1] + frame->linesize[1]* i + j);
|
||||
*(img.pMappedData + img.memory_offset_plane1 + frame->linesize[1]*2*i +j * 2+1) = *(frame->data[2] + frame->linesize[2]* i + j);
|
||||
}
|
||||
//memcpy(img.pMappedData + img.memory_offset_plane1 , frame->data[1], frame->linesize[1] * frame->height/2);
|
||||
//assert(!frame->data[2]);
|
||||
pthread_mutex_lock(&gFF.mutex);
|
||||
gFF.decode_index = idx;
|
||||
pthread_cond_signal(&gFF.cond);
|
||||
|
@ -198,51 +218,54 @@ static void *DecoderThread(void*)
|
|||
|
||||
static void SetupFFMpeg(VkInstance inst, VulkanDevice &dev)
|
||||
{
|
||||
for(int i = 0; i < 4; i++)
|
||||
for(int i = 0; i < MAX_DECODER_FRAMES; i++)
|
||||
CreateSoftwareImage(inst, dev, gFF.images[i]);
|
||||
pthread_mutex_init(&gFF.mutex, NULL);
|
||||
pthread_cond_init(&gFF.cond, NULL);
|
||||
pthread_create(&gFF.thread, NULL, &DecoderThread, NULL);
|
||||
}
|
||||
|
||||
|
||||
#define RECONSTRUCTION_TARGET_FRAMES 4
|
||||
|
||||
struct GraphicsApplicationPipeline: BaseVulkanPipeline
|
||||
{
|
||||
|
||||
void Init(VkDevice dev, VkRenderPass renderPass)
|
||||
void Init(VkDevice dev, VkRenderPass renderPass, VkSampler *pImmutableSamplers)
|
||||
{
|
||||
|
||||
device = dev;
|
||||
CreateDescriptorSetLayout(
|
||||
BasicBinding(0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,1,VK_SHADER_STAGE_VERTEX_BIT)
|
||||
//BasicBinding(0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,1,VK_SHADER_STAGE_VERTEX_BIT)
|
||||
BasicBinding(0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,1,VK_SHADER_STAGE_FRAGMENT_BIT, pImmutableSamplers)
|
||||
);
|
||||
CreatePool(1,
|
||||
BasicPoolSize(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1)
|
||||
//BasicPoolSize(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1)
|
||||
BasicPoolSize(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1)
|
||||
);
|
||||
VkShaderModule vp, fp;
|
||||
CreateGraphicsPipeline(renderPass,
|
||||
Stages(
|
||||
ShaderFromFile(vp, "triangle.vert.spv", VK_SHADER_STAGE_VERTEX_BIT),
|
||||
ShaderFromFile(fp, "triangle.frag.spv", VK_SHADER_STAGE_FRAGMENT_BIT)),
|
||||
VertexBindings(
|
||||
VertexBinding(0, sizeof(Vertex))
|
||||
),
|
||||
VertexAttributes(
|
||||
VertAttrib(0, 0, offsetof(Vertex,position)),
|
||||
VertAttrib(1, 0, offsetof(Vertex,color))
|
||||
),
|
||||
ShaderFromFile(vp, "quad.vert.spv", VK_SHADER_STAGE_VERTEX_BIT),
|
||||
ShaderFromFile(fp, "quad.frag.spv", VK_SHADER_STAGE_FRAGMENT_BIT)),
|
||||
VertexBindings(),
|
||||
VertexAttributes(),
|
||||
DynamicStates(
|
||||
VK_DYNAMIC_STATE_VIEWPORT,
|
||||
VK_DYNAMIC_STATE_SCISSOR
|
||||
),
|
||||
DepthStencil(true, true)
|
||||
DepthStencil(true, true),
|
||||
AssemblyTopology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN)
|
||||
//RasterMode(VK_FRONT_FACE_CLOCKWISE)
|
||||
);
|
||||
vkDestroyShaderModule(device, vp, NULL);
|
||||
vkDestroyShaderModule(device, fp, NULL);
|
||||
}
|
||||
void UpdateDescriptors(VkDescriptorSet dstSet, const VkDescriptorBufferInfo &buffer)
|
||||
void UpdateDescriptors(VkDescriptorSet dstSet, VkImageView imageView, VkSampler sampler)
|
||||
{
|
||||
WriteDescriptors(
|
||||
BufferWrite(dstSet, 0, buffer)
|
||||
//BufferWrite(dstSet, 0, buffer)
|
||||
ImageWrite(dstSet, 0, ImageDescriptor(imageView, VK_IMAGE_LAYOUT_GENERAL, sampler), VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
|
||||
);
|
||||
}
|
||||
};
|
||||
|
@ -257,12 +280,13 @@ struct GraphicsApplication
|
|||
VkImage swapchainImages[MAX_SWAPCHAIN_IMAGES];
|
||||
unsigned int numSwapchainImages;
|
||||
VulkanFramebuffer swapchainFbs[MAX_SWAPCHAIN_IMAGES];
|
||||
VkCommandBuffer commandBuffers[MAX_SWAPCHAIN_IMAGES];
|
||||
VkCommandBuffer commandBuffers[MAX_SWAPCHAIN_IMAGES * RECONSTRUCTION_TARGET_FRAMES];
|
||||
VkCommandBuffer reconstructionCommandBuffers[MAX_DECODER_FRAMES * RECONSTRUCTION_TARGET_FRAMES];
|
||||
VkFence chainFences[MAX_SWAPCHAIN_IMAGES];
|
||||
VkSemaphore swapchainRenderSemaphore[MAX_SWAPCHAIN_IMAGES], swapchainPresentSemaphore[MAX_SWAPCHAIN_IMAGES];
|
||||
struct UBO
|
||||
{
|
||||
glm::mat4 transformationMatrix;
|
||||
//glm::mat4 transformationMatrix;
|
||||
};
|
||||
|
||||
GraphicsApplicationPipeline graphicsPipeline;
|
||||
|
@ -364,6 +388,7 @@ struct GraphicsApplication
|
|||
static unsigned int frameNum;
|
||||
// Rotate based on time
|
||||
long long millis = frameNum++;
|
||||
#if 0
|
||||
float angle = (millis % 400) / 400.0f * glm::radians(360.f);
|
||||
|
||||
glm::mat4 modelMatrix = glm::identity<glm::mat4>();
|
||||
|
@ -377,6 +402,7 @@ struct GraphicsApplication
|
|||
// Set up projection
|
||||
auto projMatrix = glm::perspective(glm::radians(70.f), 800.0f / 600.0f, 0.1f, 10.0f);
|
||||
((UBO*)uboBuf.mapped)->transformationMatrix = projMatrix * viewMatrix * modelMatrix;
|
||||
#endif
|
||||
}
|
||||
|
||||
void waitFence(int chidx)
|
||||
|
@ -399,9 +425,10 @@ struct GraphicsApplication
|
|||
CreateWindow("demo", width, height, false);
|
||||
CreateSwapchain(width,height);
|
||||
dev.CreateAndMap(uboBuf, sizeof(UBO));
|
||||
graphicsPipeline.Init(dev.device, swapchainFbs[0].render_pass);
|
||||
SetupFFMpeg(context.instance, dev);
|
||||
graphicsPipeline.Init(dev.device, swapchainFbs[0].render_pass, &gFF.images[0].ycbcr_sampler);
|
||||
descriptorSet = graphicsPipeline.AllocateSingleDescriptorSet();
|
||||
graphicsPipeline.UpdateDescriptors(descriptorSet, uboBuf.descriptor);
|
||||
graphicsPipeline.UpdateDescriptors(descriptorSet, gFF.images[0].image_view, gFF.images[0].ycbcr_sampler);
|
||||
updateUniformData();
|
||||
|
||||
Vertex vertices[] = {
|
||||
|
@ -424,17 +451,18 @@ struct GraphicsApplication
|
|||
|
||||
vkCmdBindDescriptorSets(commandBuffers[i],VK_PIPELINE_BIND_POINT_GRAPHICS,graphicsPipeline.pipelineLayout, 0, 1, &descriptorSet, 0, nullptr);
|
||||
vkCmdBindPipeline(commandBuffers[i], VK_PIPELINE_BIND_POINT_GRAPHICS,graphicsPipeline.pipeline);
|
||||
VkDeviceSize offset = 0;
|
||||
vkCmdBindVertexBuffers(commandBuffers[i],0, 1, &stagingVert.buffer, &offset);
|
||||
vkCmdBindIndexBuffer(commandBuffers[i], stagingInd.buffer, 0, VK_INDEX_TYPE_UINT32);
|
||||
vkCmdDrawIndexed(commandBuffers[i], 3, 1, 0, 0, 0);
|
||||
// VkDeviceSize offset = 0;
|
||||
//vkCmdBindVertexBuffers(commandBuffers[i],0, 1, &stagingVert.buffer, &offset);
|
||||
//vkCmdBindIndexBuffer(commandBuffers[i], stagingInd.buffer, 0, VK_INDEX_TYPE_UINT32);
|
||||
//vkCmdDrawIndexed(commandBuffers[i], 3, 1, 0, 0, 0);
|
||||
vkCmdDraw(commandBuffers[i], 4, 1, 0, 0);
|
||||
vkCmdEndRenderPass(commandBuffers[i]);
|
||||
VulkanTexture::SetImageLayout(commandBuffers[i], swapchainImages[i], VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 });
|
||||
vkEndCommandBuffer(commandBuffers[i]);
|
||||
VK_CHECK_RESULT(CallWith(FenceInfo(VK_FENCE_CREATE_SIGNALED_BIT),
|
||||
vkCreateFence(dev.device, &ref, NULL, &chainFences[i])));
|
||||
}
|
||||
SetupFFMpeg(context.instance, dev);
|
||||
|
||||
uint32_t idx = 0;
|
||||
int sem_idx = 0;
|
||||
while(true)
|
||||
|
|
|
@ -311,13 +311,13 @@ struct BaseVulkanPipeline
|
|||
return {sampler, imageView, imageLayout};
|
||||
}
|
||||
// todo: different types
|
||||
VkWriteDescriptorSet ImageWrite(VkDescriptorSet dstSet, uint32_t binding, const VkDescriptorImageInfo &info)
|
||||
VkWriteDescriptorSet ImageWrite(VkDescriptorSet dstSet, uint32_t binding, const VkDescriptorImageInfo &info, VkDescriptorType tp = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE)
|
||||
{
|
||||
VkWriteDescriptorSet wr = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
|
||||
wr.dstSet = dstSet;
|
||||
wr.dstBinding = binding;
|
||||
wr.descriptorCount = 1;
|
||||
wr.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
||||
wr.descriptorType = tp;
|
||||
wr.pImageInfo = &info;
|
||||
return wr;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue