From 07adf898e7a97666254e5f032ba1d9f5b92e5705 Mon Sep 17 00:00:00 2001 From: Benjamin Schaaf Date: Mon, 27 Dec 2021 23:11:08 +1100 Subject: [PATCH 1/8] Improve megapixels-list-devices --- src/device.c | 18 ++++- src/device.h | 3 + tools/list_devices.c | 163 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 170 insertions(+), 14 deletions(-) diff --git a/src/device.c b/src/device.c index 032ca94..6e4f5aa 100644 --- a/src/device.c +++ b/src/device.c @@ -153,6 +153,12 @@ mp_device_close(MPDevice *device) free(device); } +int +mp_device_get_fd(const MPDevice *device) +{ + return device->fd; +} + bool mp_device_setup_link(MPDevice *device, uint32_t source_pad_id, @@ -375,6 +381,7 @@ mp_device_get_num_links(const MPDevice *device) struct _MPDeviceList { MPDevice *device; MPDeviceList *next; + char path[PATH_MAX]; }; MPDeviceList * @@ -387,8 +394,8 @@ mp_device_list_new() DIR *d = opendir("/dev"); while ((dir = readdir(d)) != NULL) { if (strncmp(dir->d_name, "media", 5) == 0) { - char path[261]; - snprintf(path, 261, "/dev/%s", dir->d_name); + char path[PATH_MAX]; + snprintf(path, PATH_MAX, "/dev/%s", dir->d_name); MPDevice *device = mp_device_open(path); @@ -396,6 +403,7 @@ mp_device_list_new() MPDeviceList *next = malloc(sizeof(MPDeviceList)); next->device = device; next->next = current; + memcpy(next->path, path, sizeof(path)); current = next; } } @@ -461,6 +469,12 @@ mp_device_list_get(const MPDeviceList *device_list) return device_list->device; } +const char * +mp_device_list_get_path(const MPDeviceList *device_list) +{ + return device_list->path; +} + MPDeviceList * mp_device_list_next(const MPDeviceList *device_list) { diff --git a/src/device.h b/src/device.h index 9ea69f1..0d6efc7 100644 --- a/src/device.h +++ b/src/device.h @@ -15,6 +15,8 @@ MPDevice *mp_device_open(const char *path); MPDevice *mp_device_new(int fd); void mp_device_close(MPDevice *device); +int mp_device_get_fd(const MPDevice *device); + bool mp_device_setup_link(MPDevice *device, uint32_t source_pad_id, uint32_t sink_pad_id, @@ -62,4 +64,5 @@ MPDevice *mp_device_list_find_remove(MPDeviceList **device_list, MPDevice *mp_device_list_remove(MPDeviceList **device_list); MPDevice *mp_device_list_get(const MPDeviceList *device_list); +const char *mp_device_list_get_path(const MPDeviceList *device_list); MPDeviceList *mp_device_list_next(const MPDeviceList *device_list); diff --git a/tools/list_devices.c b/tools/list_devices.c index e748a00..9053ee5 100644 --- a/tools/list_devices.c +++ b/tools/list_devices.c @@ -1,7 +1,117 @@ #include "device.h" +#include +#include #include #include +const char * +entity_type_str(uint32_t type) +{ + switch (type) { + case MEDIA_ENT_F_UNKNOWN: + return "UNKNOWN"; + case MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN: + return "V4L2_SUBDEV_UNKNOWN"; + case MEDIA_ENT_F_IO_V4L: + return "IO_V4L"; + case MEDIA_ENT_F_IO_VBI: + return "IO_VBI"; + case MEDIA_ENT_F_IO_SWRADIO: + return "IO_SWRADIO"; + case MEDIA_ENT_F_IO_DTV: + return "IO_DTV"; + case MEDIA_ENT_F_DTV_DEMOD: + return "DTV_DEMOD"; + case MEDIA_ENT_F_TS_DEMUX: + return "TS_DEMUX"; + case MEDIA_ENT_F_DTV_CA: + return "DTV_CA"; + case MEDIA_ENT_F_DTV_NET_DECAP: + return "DTV_NET_DECAP"; + case MEDIA_ENT_F_CAM_SENSOR: + return "CAM_SENSOR"; + case MEDIA_ENT_F_FLASH: + return "FLASH"; + case MEDIA_ENT_F_LENS: + return "LENS"; + case MEDIA_ENT_F_ATV_DECODER: + return "ATV_DECODER"; + case MEDIA_ENT_F_TUNER: + return "TUNER"; + case MEDIA_ENT_F_IF_VID_DECODER: + return "IF_VID_DECODER"; + case MEDIA_ENT_F_IF_AUD_DECODER: + return "IF_AUD_DECODER"; + case MEDIA_ENT_F_AUDIO_CAPTURE: + return "AUDIO_CAPTURE"; + case MEDIA_ENT_F_AUDIO_PLAYBACK: + return "AUDIO_PLAYBACK"; + case MEDIA_ENT_F_AUDIO_MIXER: + return "AUDIO_MIXER"; + case MEDIA_ENT_F_PROC_VIDEO_COMPOSER: + return "PROC_VIDEO_COMPOSER"; + case MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER: + return "PROC_VIDEO_PIXEL_FORMATTER"; + case MEDIA_ENT_F_PROC_VIDEO_PIXEL_ENC_CONV: + return "PROC_VIDEO_PIXEL_ENC_CONV"; + case MEDIA_ENT_F_PROC_VIDEO_LUT: + return "PROC_VIDEO_LUT"; + case MEDIA_ENT_F_PROC_VIDEO_SCALER: + return "PROC_VIDEO_SCALER"; + case MEDIA_ENT_F_PROC_VIDEO_STATISTICS: + return "PROC_VIDEO_STATISTICS"; + default: + return "invalid type"; + } +} + +const char * +intf_type_str(uint32_t type) +{ + switch (type) { + case MEDIA_INTF_T_DVB_FE: + return "DVB_FE"; + case MEDIA_INTF_T_DVB_DEMUX: + return "DVB_DEMUX"; + case MEDIA_INTF_T_DVB_DVR: + return "DVB_DVR"; + case MEDIA_INTF_T_DVB_CA: + return "DVB_CA"; + case MEDIA_INTF_T_DVB_NET: + return "DVB_NET"; + case MEDIA_INTF_T_V4L_VIDEO: + return "V4L_VIDEO"; + case MEDIA_INTF_T_V4L_VBI: + return "V4L_VBI"; + case MEDIA_INTF_T_V4L_RADIO: + return "V4L_RADIO"; + case MEDIA_INTF_T_V4L_SUBDEV: + return "V4L_SUBDEV"; + case MEDIA_INTF_T_V4L_SWRADIO: + return "V4L_SWRADIO"; + case MEDIA_INTF_T_V4L_TOUCH: + return "V4L_TOUCH"; + case MEDIA_INTF_T_ALSA_PCM_CAPTURE: + return "ALSA_PCM_CAPTURE"; + case MEDIA_INTF_T_ALSA_PCM_PLAYBACK: + return "ALSA_PCM_PLAYBACK"; + case MEDIA_INTF_T_ALSA_CONTROL: + return "ALSA_CONTROL"; + case MEDIA_INTF_T_ALSA_COMPRESS: + return "ALSA_COMPRESS"; + case MEDIA_INTF_T_ALSA_RAWMIDI: + return "ALSA_RAWMIDI"; + case MEDIA_INTF_T_ALSA_HWDEP: + return "ALSA_HWDEP"; + case MEDIA_INTF_T_ALSA_SEQUENCER: + return "ALSA_SEQUENCER"; + case MEDIA_INTF_T_ALSA_TIMER: + return "ALSA_TIMER"; + default: + return "invalid type"; + } +} + int main(int argc, char *argv[]) { @@ -12,6 +122,7 @@ main(int argc, char *argv[]) const struct media_device_info *info = mp_device_get_info(device); printf("%s (%s) %s\n", info->model, info->driver, info->serial); + printf(" Path: %s\n", mp_device_list_get_path(list)); printf(" Bus Info: %s\n", info->bus_info); printf(" Media Version: %d\n", info->media_version); printf(" HW Revision: %d\n", info->hw_revision); @@ -22,10 +133,10 @@ main(int argc, char *argv[]) size_t num = mp_device_get_num_entities(device); printf(" Entities (%ld):\n", num); for (int i = 0; i < num; ++i) { - printf(" %d %s (%d)\n", + printf(" %d %s (%s)\n", entities[i].id, entities[i].name, - entities[i].function); + entity_type_str(entities[i].function)); } const struct media_v2_interface *interfaces = @@ -33,33 +144,61 @@ main(int argc, char *argv[]) num = mp_device_get_num_interfaces(device); printf(" Interfaces (%ld):\n", num); for (int i = 0; i < num; ++i) { - printf(" %d (%d - %d) devnode %d:%d\n", + // Unused + assert(interfaces[i].flags == 0); + + char buf[PATH_MAX]; + buf[0] = '\0'; + mp_find_device_path(interfaces[i].devnode, buf, PATH_MAX); + + printf(" %d (%s) devnode %d:%d %s\n", interfaces[i].id, - interfaces[i].intf_type, - interfaces[i].flags, + intf_type_str(interfaces[i].intf_type), interfaces[i].devnode.major, - interfaces[i].devnode.minor); + interfaces[i].devnode.minor, + buf); } const struct media_v2_pad *pads = mp_device_get_pads(device); num = mp_device_get_num_pads(device); printf(" Pads (%ld):\n", num); for (int i = 0; i < num; ++i) { - printf(" %d for device:%d (%d)\n", + printf(" %d for device:%d (", pads[i].id, - pads[i].entity_id, - pads[i].flags); + pads[i].entity_id); + + if (pads[i].flags & MEDIA_PAD_FL_SINK) + printf("SINK "); + if (pads[i].flags & MEDIA_PAD_FL_SOURCE) + printf("SOURCE "); + if (pads[i].flags & MEDIA_PAD_FL_MUST_CONNECT) + printf("MUST_CONNECT "); + printf(")\n"); } const struct media_v2_link *links = mp_device_get_links(device); num = mp_device_get_num_links(device); printf(" Links (%ld):\n", num); for (int i = 0; i < num; ++i) { - printf(" %d from:%d to:%d (%d)\n", + printf(" %d from:%d to:%d (", links[i].id, links[i].source_id, - links[i].sink_id, - links[i].flags); + links[i].sink_id); + + if (links[i].flags & MEDIA_LNK_FL_ENABLED) + printf("ENABLED "); + if (links[i].flags & MEDIA_LNK_FL_IMMUTABLE) + printf("IMMUTABLE "); + if (links[i].flags & MEDIA_LNK_FL_DYNAMIC) + printf("DYNAMIC "); + + uint32_t type = links[i].flags & MEDIA_LNK_FL_LINK_TYPE; + if (type == MEDIA_LNK_FL_INTERFACE_LINK) { + printf("INTERFACE)\n"); + } else { + assert(type == MEDIA_LNK_FL_DATA_LINK); + printf("DATA)\n"); + } } list = mp_device_list_next(list); From 868291e0c6c3bea1b592f529631e3b8ec926dda3 Mon Sep 17 00:00:00 2001 From: Benjamin Schaaf Date: Sun, 9 Jan 2022 21:02:09 +1100 Subject: [PATCH 2/8] Possible fix for mplane support --- src/camera.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/camera.c b/src/camera.c index f429b73..2f88ffc 100644 --- a/src/camera.c +++ b/src/camera.c @@ -264,7 +264,6 @@ mp_camera_new(int video_fd, int subdev_fd) bool use_mplane; if (cap.capabilities & V4L2_CAP_VIDEO_CAPTURE_MPLANE) { use_mplane = true; - printf("!!\n"); } else if (cap.capabilities & V4L2_CAP_VIDEO_CAPTURE) { use_mplane = false; } else { @@ -394,10 +393,19 @@ mp_camera_get_subdev_fd(MPCamera *camera) return camera->subdev_fd; } +static enum v4l2_buf_type +get_buf_type(MPCamera *camera) +{ + if (camera->use_mplane) { + return V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + } + return V4L2_BUF_TYPE_VIDEO_CAPTURE; +} + static bool camera_mode_impl(MPCamera *camera, int request, MPCameraMode *mode) { - uint32_t pixfmt = mp_pixel_format_from_v4l_pixel_format(mode->pixel_format); + uint32_t pixfmt = mp_pixel_format_to_v4l_pixel_format(mode->pixel_format); struct v4l2_format fmt = {}; if (camera->use_mplane) { fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; @@ -460,7 +468,6 @@ mp_camera_set_mode(MPCamera *camera, MPCameraMode *mode) VIDIOC_SUBDEV_S_FRAME_INTERVAL, &interval) == -1) { errno_printerr("VIDIOC_SUBDEV_S_FRAME_INTERVAL"); - return false; } bool did_set_frame_rate = interval.interval.numerator == @@ -522,10 +529,7 @@ mp_camera_start_capture(MPCamera *camera) g_return_val_if_fail(camera->has_set_mode, false); g_return_val_if_fail(camera->num_buffers == 0, false); - enum v4l2_buf_type buftype = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (camera->use_mplane) { - buftype = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - } + const enum v4l2_buf_type buftype = get_buf_type(camera); // Start by requesting buffers struct v4l2_requestbuffers req = {}; @@ -588,7 +592,7 @@ mp_camera_start_capture(MPCamera *camera) } struct v4l2_exportbuffer expbuf = { - .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, + .type = buftype, .index = i, }; if (xioctl(camera->video_fd, VIDIOC_EXPBUF, &expbuf) == -1) { @@ -627,7 +631,7 @@ mp_camera_start_capture(MPCamera *camera) } // Start capture - enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + enum v4l2_buf_type type = buftype; if (xioctl(camera->video_fd, VIDIOC_STREAMON, &type) == -1) { errno_printerr("VIDIOC_STREAMON"); goto error; @@ -653,7 +657,7 @@ error: { struct v4l2_requestbuffers req = {}; req.count = 0; - req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + req.type = buftype; req.memory = V4L2_MEMORY_MMAP; if (xioctl(camera->video_fd, VIDIOC_REQBUFS, &req) == -1) { @@ -669,7 +673,9 @@ mp_camera_stop_capture(MPCamera *camera) { g_return_val_if_fail(camera->num_buffers > 0, false); - enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + const enum v4l2_buf_type buftype = get_buf_type(camera); + + enum v4l2_buf_type type = buftype; if (xioctl(camera->video_fd, VIDIOC_STREAMOFF, &type) == -1) { errno_printerr("VIDIOC_STREAMOFF"); } @@ -690,7 +696,7 @@ mp_camera_stop_capture(MPCamera *camera) struct v4l2_requestbuffers req = {}; req.count = 0; - req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + req.type = buftype; req.memory = V4L2_MEMORY_MMAP; if (xioctl(camera->video_fd, VIDIOC_REQBUFS, &req) == -1) { errno_printerr("VIDIOC_REQBUFS"); @@ -708,8 +714,10 @@ mp_camera_is_capturing(MPCamera *camera) bool mp_camera_capture_buffer(MPCamera *camera, MPBuffer *buffer) { + const enum v4l2_buf_type buftype = get_buf_type(camera); + struct v4l2_buffer buf = {}; - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.type = buftype; buf.memory = V4L2_MEMORY_MMAP; struct v4l2_plane planes[1]; @@ -757,8 +765,10 @@ mp_camera_capture_buffer(MPCamera *camera, MPBuffer *buffer) bool mp_camera_release_buffer(MPCamera *camera, uint32_t buffer_index) { + const enum v4l2_buf_type buftype = get_buf_type(camera); + struct v4l2_buffer buf = {}; - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.type = buftype; buf.memory = V4L2_MEMORY_MMAP; buf.index = buffer_index; if (xioctl(camera->video_fd, VIDIOC_QBUF, &buf) == -1) { @@ -863,12 +873,14 @@ get_subdev_modes(MPCamera *camera, bool (*check)(MPCamera *, MPCameraMode *)) static MPCameraModeList * get_video_modes(MPCamera *camera, bool (*check)(MPCamera *, MPCameraMode *)) { + const enum v4l2_buf_type buftype = get_buf_type(camera); + MPCameraModeList *item = NULL; for (uint32_t fmt_index = 0;; ++fmt_index) { struct v4l2_fmtdesc fmt = {}; fmt.index = fmt_index; - fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + fmt.type = buftype; if (xioctl(camera->video_fd, VIDIOC_ENUM_FMT, &fmt) == -1) { if (errno != EINVAL) { errno_printerr("VIDIOC_ENUM_FMT"); From b980eb5aeaaaeb24379048f3e5c1caf6f7c69f5b Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Sat, 15 Jan 2022 15:18:42 +0400 Subject: [PATCH 3/8] Add mplanes to mp_camera_release_buffer --- src/camera.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/camera.c b/src/camera.c index 2f88ffc..7800304 100644 --- a/src/camera.c +++ b/src/camera.c @@ -771,6 +771,13 @@ mp_camera_release_buffer(MPCamera *camera, uint32_t buffer_index) buf.type = buftype; buf.memory = V4L2_MEMORY_MMAP; buf.index = buffer_index; + + struct v4l2_plane planes[1]; + if (camera->use_mplane) { + buf.m.planes = planes; + buf.length = 1; + } + if (xioctl(camera->video_fd, VIDIOC_QBUF, &buf) == -1) { errno_printerr("VIDIOC_QBUF"); return false; From f8ce7abe98b6225c54ccfc1d58e63c5478d121fc Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Sun, 9 Jan 2022 15:02:35 +0400 Subject: [PATCH 4/8] zbar_pipeline: Add support for 10-bit packed format processing --- src/zbar_pipeline.c | 47 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/src/zbar_pipeline.c b/src/zbar_pipeline.c index 25d6a44..b231557 100644 --- a/src/zbar_pipeline.c +++ b/src/zbar_pipeline.c @@ -174,7 +174,11 @@ process_image(MPPipeline *pipeline, MPZBarImage **_image) assert(image->pixel_format == MP_PIXEL_FMT_BGGR8 || image->pixel_format == MP_PIXEL_FMT_GBRG8 || image->pixel_format == MP_PIXEL_FMT_GRBG8 || - image->pixel_format == MP_PIXEL_FMT_RGGB8); + image->pixel_format == MP_PIXEL_FMT_RGGB8 || + image->pixel_format == MP_PIXEL_FMT_BGGR10P || + image->pixel_format == MP_PIXEL_FMT_GBRG10P || + image->pixel_format == MP_PIXEL_FMT_GRBG10P || + image->pixel_format == MP_PIXEL_FMT_RGGB10P); // Create a grayscale image for scanning from the current preview. // Rotate/mirror correctly. @@ -182,11 +186,46 @@ process_image(MPPipeline *pipeline, MPZBarImage **_image) int height = image->height / 2; uint8_t *data = malloc(width * height * sizeof(uint8_t)); + size_t row_length = + mp_pixel_format_width_to_bytes(image->pixel_format, image->width); size_t i = 0; - for (int y = 0; y < image->height; y += 2) { - for (int x = 0; x < image->width; x += 2) { - data[i++] = image->data[x + image->width * y]; + size_t offset; + switch (image->pixel_format) { + case MP_PIXEL_FMT_BGGR8: + case MP_PIXEL_FMT_GBRG8: + case MP_PIXEL_FMT_GRBG8: + case MP_PIXEL_FMT_RGGB8: + for (int y = 0; y < image->height; y += 2) { + for (int x = 0; x < row_length; x += 2) { + data[i++] = image->data[x + row_length * y]; + } } + break; + case MP_PIXEL_FMT_BGGR10P: + case MP_PIXEL_FMT_GBRG10P: + case MP_PIXEL_FMT_GRBG10P: + case MP_PIXEL_FMT_RGGB10P: + // Skip 5th byte of each 4-pixel segment by incrementing an + // offset every time a 5th byte is reached, making the + // X coordinate land on the next byte: + // + // image->data | | | | X | | | | X | | | | X | | | | X | ... + // x 0 2 4 6 8 10 12 14 16 18 20 ... + // offset 0 1 2 3 4 5 ... + // > ---> -----> -------> + // x + offset 0 2 4 6 8 10 12 16 18 ... + for (int y = 0; y < image->height; y += 2) { + offset = 0; + for (int x = 0; x < image->width; x += 2) { + if (x % 4 == 0) + offset += 1; + + data[i++] = image->data[x + offset + row_length * y]; + } + } + break; + default: + assert(0); } // Create image for zbar From c16dbf68101f2d72c331553570dd112b60751088 Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Sun, 16 Jan 2022 19:51:54 +0400 Subject: [PATCH 5/8] Add support for BGGR10P debayering --- data/debayer.frag | 25 ++++++++++++++++++++++++- src/gles2_debayer.c | 28 +++++++++++++++++++++++++--- src/process_pipeline.c | 2 +- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/data/debayer.frag b/data/debayer.frag index 77b9097..e7583eb 100644 --- a/data/debayer.frag +++ b/data/debayer.frag @@ -1,25 +1,48 @@ #ifdef GL_ES -precision mediump float; +precision highp float; #endif uniform sampler2D texture; uniform mat3 color_matrix; +#ifdef BITS_10 +uniform float row_length; +#endif varying vec2 top_left_uv; varying vec2 top_right_uv; varying vec2 bottom_left_uv; varying vec2 bottom_right_uv; +#ifdef BITS_10 +vec2 +skip_5th_pixel(vec2 uv) +{ + vec2 new_uv = uv; + + new_uv.x *= 0.8; + new_uv.x += floor(uv.x * row_length / 5.0) / row_length; + + return new_uv; +} +#endif + void main() { // Note the coordinates for texture samples need to be a varying, as the // Mali-400 has this as a fast path allowing 32-bit floats. Otherwise // they end up as 16-bit floats and that's not accurate enough. +#ifdef BITS_10 + vec4 samples = vec4(texture2D(texture, skip_5th_pixel(top_left_uv)).r, + texture2D(texture, skip_5th_pixel(top_right_uv)).r, + texture2D(texture, skip_5th_pixel(bottom_left_uv)).r, + texture2D(texture, skip_5th_pixel(bottom_right_uv)).r); +#else vec4 samples = vec4(texture2D(texture, top_left_uv).r, texture2D(texture, top_right_uv).r, texture2D(texture, bottom_left_uv).r, texture2D(texture, bottom_right_uv).r); +#endif // Assume BGGR for now. Currently this just takes 3 of the four samples // for each pixel, there's room here to do some better debayering. diff --git a/src/gles2_debayer.c b/src/gles2_debayer.c index 7f5a9bd..f7400c2 100644 --- a/src/gles2_debayer.c +++ b/src/gles2_debayer.c @@ -8,12 +8,15 @@ #define TEX_COORD_ATTRIBUTE 1 struct _GLES2Debayer { + MPPixelFormat format; + GLuint frame_buffer; GLuint program; GLuint uniform_transform; GLuint uniform_pixel_size; GLuint uniform_texture; GLuint uniform_color_matrix; + GLuint uniform_row_length; GLuint quad; }; @@ -21,7 +24,7 @@ struct _GLES2Debayer { GLES2Debayer * gles2_debayer_new(MPPixelFormat format) { - if (format != MP_PIXEL_FMT_BGGR8) { + if (format != MP_PIXEL_FMT_BGGR8 && format != MP_PIXEL_FMT_BGGR10P) { return NULL; } @@ -29,6 +32,14 @@ gles2_debayer_new(MPPixelFormat format) glGenFramebuffers(1, &frame_buffer); check_gl(); + char format_def[32]; + snprintf(format_def, + 32, + "#define BITS_%d\n", + mp_pixel_format_bits_per_pixel(format)); + + const GLchar *def[1] = { format_def }; + GLuint shaders[] = { gl_util_load_shader("/org/postmarketos/Megapixels/debayer.vert", GL_VERTEX_SHADER, @@ -36,8 +47,8 @@ gles2_debayer_new(MPPixelFormat format) 0), gl_util_load_shader("/org/postmarketos/Megapixels/debayer.frag", GL_FRAGMENT_SHADER, - NULL, - 0), + def, + 1), }; GLuint program = gl_util_link_program(shaders, 2); @@ -46,6 +57,8 @@ gles2_debayer_new(MPPixelFormat format) check_gl(); GLES2Debayer *self = malloc(sizeof(GLES2Debayer)); + self->format = format; + self->frame_buffer = frame_buffer; self->program = program; @@ -54,6 +67,9 @@ gles2_debayer_new(MPPixelFormat format) self->uniform_texture = glGetUniformLocation(self->program, "texture"); self->uniform_color_matrix = glGetUniformLocation(self->program, "color_matrix"); + if (mp_pixel_format_bits_per_pixel(self->format) == 10) + self->uniform_row_length = + glGetUniformLocation(self->program, "row_length"); check_gl(); self->quad = gl_util_new_quad(); @@ -135,6 +151,12 @@ gles2_debayer_configure(GLES2Debayer *self, self->uniform_color_matrix, 1, GL_FALSE, identity); } check_gl(); + + if (mp_pixel_format_bits_per_pixel(self->format) == 10) { + assert(src_width % 4 == 0); + glUniform1f(self->uniform_row_length, src_width + src_width / 4); + check_gl(); + } } void diff --git a/src/process_pipeline.c b/src/process_pipeline.c index 65b10c5..53d70b2 100644 --- a/src/process_pipeline.c +++ b/src/process_pipeline.c @@ -297,7 +297,7 @@ process_image_for_preview(const uint8_t *image) glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, - mode.width, + mp_pixel_format_width_to_bytes(mode.pixel_format, mode.width), mode.height, 0, GL_LUMINANCE, From 212e75ed3c0103d5d9c4fd44bec985190d32452e Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Mon, 17 Jan 2022 17:05:51 +0400 Subject: [PATCH 6/8] Add debayering support for all bayer CFAs --- README.md | 2 +- data/debayer.frag | 10 ++++++++-- src/camera.c | 32 ++++++++++++++++++++++++++++++++ src/camera.h | 1 + src/gles2_debayer.c | 12 ++++++++---- src/process_pipeline.c | 22 +++++++++++++++------- 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index daf7c6a..3bda179 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ when previewing. * `width=640` and `height=480` the resolution to use for the sensor * `rate=15` the refresh rate in fps to use for the sensor -* `fmt=BGGR8` sets the pixel and bus formats used when capturing from the sensor, only BGGR8 is fully supported +* `fmt=BGGR8` sets the pixel and bus formats used when capturing from the sensor. # Post processing diff --git a/data/debayer.frag b/data/debayer.frag index e7583eb..bf3d042 100644 --- a/data/debayer.frag +++ b/data/debayer.frag @@ -44,9 +44,15 @@ main() texture2D(texture, bottom_right_uv).r); #endif - // Assume BGGR for now. Currently this just takes 3 of the four samples - // for each pixel, there's room here to do some better debayering. +#if defined(CFA_BGGR) vec3 color = vec3(samples.w, (samples.y + samples.z) / 2.0, samples.x); +#elif defined(CFA_GBRG) + vec3 color = vec3(samples.z, (samples.x + samples.w) / 2.0, samples.y); +#elif defined(CFA_GRBG) + vec3 color = vec3(samples.y, (samples.x + samples.w) / 2.0, samples.z); +#else + vec3 color = vec3(samples.x, (samples.y + samples.z) / 2.0, samples.w); +#endif // Some crude blacklevel correction to make the preview a bit nicer, this // should be an uniform diff --git a/src/camera.c b/src/camera.c index 7800304..0c3e7f0 100644 --- a/src/camera.c +++ b/src/camera.c @@ -144,6 +144,38 @@ mp_pixel_format_pixel_depth(MPPixelFormat pixel_format) } } +const char * +mp_pixel_format_cfa(MPPixelFormat pixel_format) +{ + g_return_val_if_fail(pixel_format < MP_PIXEL_FMT_MAX, 0); + switch (pixel_format) { + case MP_PIXEL_FMT_BGGR8: + case MP_PIXEL_FMT_BGGR10P: + return "BGGR"; + break; + case MP_PIXEL_FMT_GBRG8: + case MP_PIXEL_FMT_GBRG10P: + return "GBRG"; + break; + case MP_PIXEL_FMT_GRBG8: + case MP_PIXEL_FMT_GRBG10P: + return "GRBG"; + break; + case MP_PIXEL_FMT_RGGB8: + case MP_PIXEL_FMT_RGGB10P: + return "RGGB"; + break; + case MP_PIXEL_FMT_UYVY: + return "UYUV"; + break; + case MP_PIXEL_FMT_YUYV: + return "YUYV"; + break; + default: + return "unsupported"; + } +} + uint32_t mp_pixel_format_width_to_bytes(MPPixelFormat pixel_format, uint32_t width) { diff --git a/src/camera.h b/src/camera.h index 5b8dc12..60fb70f 100644 --- a/src/camera.h +++ b/src/camera.h @@ -31,6 +31,7 @@ uint32_t mp_pixel_format_to_v4l_bus_code(MPPixelFormat pixel_format); uint32_t mp_pixel_format_bits_per_pixel(MPPixelFormat pixel_format); uint32_t mp_pixel_format_pixel_depth(MPPixelFormat pixel_format); +const char *mp_pixel_format_cfa(MPPixelFormat pixel_format); uint32_t mp_pixel_format_width_to_bytes(MPPixelFormat pixel_format, uint32_t width); uint32_t mp_pixel_format_width_to_colors(MPPixelFormat pixel_format, uint32_t width); uint32_t mp_pixel_format_height_to_colors(MPPixelFormat pixel_format, diff --git a/src/gles2_debayer.c b/src/gles2_debayer.c index f7400c2..f266d7f 100644 --- a/src/gles2_debayer.c +++ b/src/gles2_debayer.c @@ -24,7 +24,10 @@ struct _GLES2Debayer { GLES2Debayer * gles2_debayer_new(MPPixelFormat format) { - if (format != MP_PIXEL_FMT_BGGR8 && format != MP_PIXEL_FMT_BGGR10P) { + if (format != MP_PIXEL_FMT_BGGR8 && format != MP_PIXEL_FMT_GBRG8 && + format != MP_PIXEL_FMT_GRBG8 && format != MP_PIXEL_FMT_RGGB8 && + format != MP_PIXEL_FMT_BGGR10P && format != MP_PIXEL_FMT_GBRG10P && + format != MP_PIXEL_FMT_GRBG10P && format != MP_PIXEL_FMT_RGGB10P) { return NULL; } @@ -32,10 +35,11 @@ gles2_debayer_new(MPPixelFormat format) glGenFramebuffers(1, &frame_buffer); check_gl(); - char format_def[32]; + char format_def[64]; snprintf(format_def, - 32, - "#define BITS_%d\n", + 64, + "#define CFA_%s\n#define BITS_%d\n", + mp_pixel_format_cfa(format), mp_pixel_format_bits_per_pixel(format)); const GLchar *def[1] = { format_def }; diff --git a/src/process_pipeline.c b/src/process_pipeline.c index 53d70b2..6edadbd 100644 --- a/src/process_pipeline.c +++ b/src/process_pipeline.c @@ -224,14 +224,9 @@ init_gl(MPPipeline *pipeline, GdkSurface **surface) check_gl(); } - gles2_debayer = gles2_debayer_new(MP_PIXEL_FMT_BGGR8); - check_gl(); - glPixelStorei(GL_UNPACK_ALIGNMENT, 1); check_gl(); - gles2_debayer_use(gles2_debayer); - for (size_t i = 0; i < NUM_BUFFERS; ++i) { glGenTextures(1, &output_buffers[i].texture_id); glBindTexture(GL_TEXTURE_2D, output_buffers[i].texture_id); @@ -718,7 +713,7 @@ mp_process_pipeline_capture() } static void -on_output_changed() +on_output_changed(bool format_changed) { output_buffer_width = mode.width / 2; output_buffer_height = mode.height / 2; @@ -744,6 +739,17 @@ on_output_changed() glBindTexture(GL_TEXTURE_2D, 0); + // Create new gles2_debayer on format change + if (format_changed) { + if (gles2_debayer) + gles2_debayer_free(gles2_debayer); + + gles2_debayer = gles2_debayer_new(mode.pixel_format); + check_gl(); + + gles2_debayer_use(gles2_debayer); + } + gles2_debayer_configure( gles2_debayer, output_buffer_width, @@ -772,6 +778,8 @@ update_state(MPPipeline *pipeline, const struct mp_process_pipeline_state *state preview_height != state->preview_height || device_rotation != state->device_rotation; + const bool format_changed = mode.pixel_format != state->mode.pixel_format; + camera = state->camera; mode = state->mode; @@ -793,7 +801,7 @@ update_state(MPPipeline *pipeline, const struct mp_process_pipeline_state *state if (output_changed) { camera_rotation = mod(camera->rotate - device_rotation, 360); - on_output_changed(); + on_output_changed(format_changed); } struct mp_main_state main_state = { From d8f34f7f893822d7a6895e00e04c2d0583935d06 Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Fri, 21 Jan 2022 13:28:01 +0400 Subject: [PATCH 7/8] Fix 10-bit format capture TIFF/DNG need a different packing format for 10-bit images. Add a function to repack the raw sensor data into the required sequencial format. --- src/process_pipeline.c | 48 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/process_pipeline.c b/src/process_pipeline.c index 6edadbd..7838d45 100644 --- a/src/process_pipeline.c +++ b/src/process_pipeline.c @@ -174,6 +174,37 @@ mp_process_pipeline_buffer_get_texture_id(MPProcessPipelineBuffer *buf) return buf->texture_id; } +static void +repack_image_sequencial(const uint8_t *src_buf, + uint8_t *dst_buf, + size_t width, + size_t height) +{ + uint16_t pixels[4]; + + /* + * Repack 40 bits stored in sensor format into sequencial format + * + * src_buf: 11111111 22222222 33333333 44444444 11223344 ... + * dst_buf: 11111111 11222222 22223333 33333344 44444444 ... + */ + assert(width % 4 == 0); + for (size_t i = 0; i < (width + width / 4) * height; i += 5) { + /* Extract pixels from packed sensor format */ + pixels[0] = (src_buf[i] << 2) | (src_buf[i + 4] >> 6); + pixels[1] = (src_buf[i + 1] << 2) | (src_buf[i + 4] >> 4 & 0x03); + pixels[2] = (src_buf[i + 2] << 2) | (src_buf[i + 4] >> 2 & 0x03); + pixels[3] = (src_buf[i + 3] << 2) | (src_buf[i + 4] & 0x03); + + /* Pack pixels into sequencial format */ + dst_buf[i] = (pixels[0] >> 2 & 0xff); + dst_buf[i + 1] = (pixels[0] << 6 & 0xff) | (pixels[1] >> 4 & 0x3f); + dst_buf[i + 2] = (pixels[1] << 4 & 0xff) | (pixels[2] >> 6 & 0x0f); + dst_buf[i + 3] = (pixels[2] << 2 & 0xff) | (pixels[3] >> 8 & 0x03); + dst_buf[i + 4] = (pixels[3] & 0xff); + } +} + static GLES2Debayer *gles2_debayer = NULL; static GdkGLContext *context; @@ -475,10 +506,22 @@ process_image_for_capture(const uint8_t *image, int count) TIFFCheckpointDirectory(tif); printf("Writing frame to %s\n", fname); + uint8_t *output_image = (uint8_t *)image; + + // Repack 10-bit image from sensor format into a sequencial format + if (mp_pixel_format_bits_per_pixel(mode.pixel_format) == 10) { + output_image = malloc(mp_pixel_format_width_to_bytes( + mode.pixel_format, mode.width) * + mode.height); + + repack_image_sequencial( + image, output_image, mode.width, mode.height); + } + for (int row = 0; row < mode.height; row++) { TIFFWriteScanline( tif, - (void *)image + + (void *)output_image + (row * mp_pixel_format_width_to_bytes( mode.pixel_format, mode.width)), row, @@ -486,6 +529,9 @@ process_image_for_capture(const uint8_t *image, int count) } TIFFWriteDirectory(tif); + if (output_image != image) + free(output_image); + // Add an EXIF block to the tiff TIFFCreateEXIFDirectory(tif); // 1 = manual, 2 = full auto, 3 = aperture priority, 4 = shutter priority From 7b403f2e7914f736289ad5d07fd1077bb7c0255f Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Fri, 21 Jan 2022 13:36:13 +0400 Subject: [PATCH 8/8] Set TIFFTAG_CFAPATTERN depending on pixel format Add a function to get a CFA pattern string that matches a given pixel format, and use it to set TIFFTAG_CFAPATTERN on capture. --- src/camera.c | 26 ++++++++++++++++++++++++++ src/camera.h | 1 + src/process_pipeline.c | 9 +++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/camera.c b/src/camera.c index 0c3e7f0..9e364b3 100644 --- a/src/camera.c +++ b/src/camera.c @@ -176,6 +176,32 @@ mp_pixel_format_cfa(MPPixelFormat pixel_format) } } +const char * +mp_pixel_format_cfa_pattern(MPPixelFormat pixel_format) +{ + g_return_val_if_fail(pixel_format < MP_PIXEL_FMT_MAX, 0); + switch (pixel_format) { + case MP_PIXEL_FMT_BGGR8: + case MP_PIXEL_FMT_BGGR10P: + return "\002\001\001\000"; + break; + case MP_PIXEL_FMT_GBRG8: + case MP_PIXEL_FMT_GBRG10P: + return "\001\002\000\001"; + break; + case MP_PIXEL_FMT_GRBG8: + case MP_PIXEL_FMT_GRBG10P: + return "\001\000\002\001"; + break; + case MP_PIXEL_FMT_RGGB8: + case MP_PIXEL_FMT_RGGB10P: + return "\000\001\001\002"; + break; + default: + return NULL; + } +} + uint32_t mp_pixel_format_width_to_bytes(MPPixelFormat pixel_format, uint32_t width) { diff --git a/src/camera.h b/src/camera.h index 60fb70f..62f15df 100644 --- a/src/camera.h +++ b/src/camera.h @@ -32,6 +32,7 @@ uint32_t mp_pixel_format_to_v4l_bus_code(MPPixelFormat pixel_format); uint32_t mp_pixel_format_bits_per_pixel(MPPixelFormat pixel_format); uint32_t mp_pixel_format_pixel_depth(MPPixelFormat pixel_format); const char *mp_pixel_format_cfa(MPPixelFormat pixel_format); +const char *mp_pixel_format_cfa_pattern(MPPixelFormat pixel_format); uint32_t mp_pixel_format_width_to_bytes(MPPixelFormat pixel_format, uint32_t width); uint32_t mp_pixel_format_width_to_colors(MPPixelFormat pixel_format, uint32_t width); uint32_t mp_pixel_format_height_to_colors(MPPixelFormat pixel_format, diff --git a/src/process_pipeline.c b/src/process_pipeline.c index 7838d45..3f23378 100644 --- a/src/process_pipeline.c +++ b/src/process_pipeline.c @@ -488,9 +488,14 @@ process_image_for_capture(const uint8_t *image, int count) static const short cfapatterndim[] = { 2, 2 }; TIFFSetField(tif, TIFFTAG_CFAREPEATPATTERNDIM, cfapatterndim); #if (TIFFLIB_VERSION < 20201219) && !LIBTIFF_CFA_PATTERN - TIFFSetField(tif, TIFFTAG_CFAPATTERN, "\002\001\001\000"); // BGGR + TIFFSetField(tif, + TIFFTAG_CFAPATTERN, + mp_pixel_format_cfa_pattern(mode.pixel_format)); #else - TIFFSetField(tif, TIFFTAG_CFAPATTERN, 4, "\002\001\001\000"); // BGGR + TIFFSetField(tif, + TIFFTAG_CFAPATTERN, + 4, + mp_pixel_format_cfa_pattern(mode.pixel_format)); #endif printf("TIFF version %d\n", TIFFLIB_VERSION); int whitelevel = camera->whitelevel;