9e10811bb5
Add support for 24-bit and 32-bit RGB formats. The fragment samples the
texture and reorders the components, using a pattern set through the
RGB_PATTERN macro. The pattern stores the shader vec4 element indices
(named {r, g, b, a} by convention, for elements 0 to 3) to be extracted
from the texture samples, when interpreted by OpenGL as RGBA.
Note that, as textures are created with GL_UNSIGNED_BYTE, the RGBA
order corresponds to bytes in memory, while the libcamera formats are
named based on the components order in a 32-bit word stored in memory in
little endian format.
An alternative to manual reordering in the shader would be to set the
texture swizzling mask. This is however not available in OpenGL ES
before version 3.0, which we don't mandate at the moment.
Only the BGR888 and RGB888 formats have been tested, with the vimc
pipeline handler.
Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Reviewed-by: Andrey Konovalov <andrey.konovalov@linaro.org>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>
580 lines
14 KiB
C++
580 lines
14 KiB
C++
/* SPDX-License-Identifier: LGPL-2.1-or-later */
|
|
/*
|
|
* Copyright (C) 2020, Linaro
|
|
*
|
|
* viewfinderGL.cpp - OpenGL Viewfinder for rendering by OpenGL shader
|
|
*/
|
|
|
|
#include "viewfinder_gl.h"
|
|
|
|
#include <QByteArray>
|
|
#include <QFile>
|
|
#include <QImage>
|
|
|
|
#include <libcamera/formats.h>
|
|
|
|
static const QList<libcamera::PixelFormat> supportedFormats{
|
|
/* YUV - packed (single plane) */
|
|
libcamera::formats::UYVY,
|
|
libcamera::formats::VYUY,
|
|
libcamera::formats::YUYV,
|
|
libcamera::formats::YVYU,
|
|
/* YUV - semi planar (two planes) */
|
|
libcamera::formats::NV12,
|
|
libcamera::formats::NV21,
|
|
libcamera::formats::NV16,
|
|
libcamera::formats::NV61,
|
|
libcamera::formats::NV24,
|
|
libcamera::formats::NV42,
|
|
/* YUV - fully planar (three planes) */
|
|
libcamera::formats::YUV420,
|
|
libcamera::formats::YVU420,
|
|
/* RGB */
|
|
libcamera::formats::ABGR8888,
|
|
libcamera::formats::ARGB8888,
|
|
libcamera::formats::BGRA8888,
|
|
libcamera::formats::RGBA8888,
|
|
libcamera::formats::BGR888,
|
|
libcamera::formats::RGB888,
|
|
};
|
|
|
|
ViewFinderGL::ViewFinderGL(QWidget *parent)
|
|
: QOpenGLWidget(parent), buffer_(nullptr), data_(nullptr),
|
|
vertexBuffer_(QOpenGLBuffer::VertexBuffer)
|
|
{
|
|
}
|
|
|
|
ViewFinderGL::~ViewFinderGL()
|
|
{
|
|
removeShader();
|
|
}
|
|
|
|
const QList<libcamera::PixelFormat> &ViewFinderGL::nativeFormats() const
|
|
{
|
|
return supportedFormats;
|
|
}
|
|
|
|
int ViewFinderGL::setFormat(const libcamera::PixelFormat &format,
|
|
const QSize &size)
|
|
{
|
|
if (format != format_) {
|
|
/*
|
|
* If the fragment already exists, remove it and create a new
|
|
* one for the new format.
|
|
*/
|
|
if (shaderProgram_.isLinked()) {
|
|
shaderProgram_.release();
|
|
shaderProgram_.removeShader(fragmentShader_.get());
|
|
fragmentShader_.reset();
|
|
}
|
|
|
|
if (!selectFormat(format))
|
|
return -1;
|
|
|
|
format_ = format;
|
|
}
|
|
|
|
size_ = size;
|
|
|
|
updateGeometry();
|
|
return 0;
|
|
}
|
|
|
|
void ViewFinderGL::stop()
|
|
{
|
|
if (buffer_) {
|
|
renderComplete(buffer_);
|
|
buffer_ = nullptr;
|
|
}
|
|
}
|
|
|
|
QImage ViewFinderGL::getCurrentImage()
|
|
{
|
|
QMutexLocker locker(&mutex_);
|
|
|
|
return grabFramebuffer();
|
|
}
|
|
|
|
void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map)
|
|
{
|
|
if (buffer->planes().size() != 1) {
|
|
qWarning() << "Multi-planar buffers are not supported";
|
|
return;
|
|
}
|
|
|
|
if (buffer_)
|
|
renderComplete(buffer_);
|
|
|
|
data_ = static_cast<unsigned char *>(map->memory);
|
|
update();
|
|
buffer_ = buffer;
|
|
}
|
|
|
|
bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
|
|
{
|
|
bool ret = true;
|
|
|
|
fragmentShaderDefines_.clear();
|
|
|
|
switch (format) {
|
|
case libcamera::formats::NV12:
|
|
horzSubSample_ = 2;
|
|
vertSubSample_ = 2;
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_UV");
|
|
fragmentShaderFile_ = ":YUV_2_planes.frag";
|
|
break;
|
|
case libcamera::formats::NV21:
|
|
horzSubSample_ = 2;
|
|
vertSubSample_ = 2;
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_VU");
|
|
fragmentShaderFile_ = ":YUV_2_planes.frag";
|
|
break;
|
|
case libcamera::formats::NV16:
|
|
horzSubSample_ = 2;
|
|
vertSubSample_ = 1;
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_UV");
|
|
fragmentShaderFile_ = ":YUV_2_planes.frag";
|
|
break;
|
|
case libcamera::formats::NV61:
|
|
horzSubSample_ = 2;
|
|
vertSubSample_ = 1;
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_VU");
|
|
fragmentShaderFile_ = ":YUV_2_planes.frag";
|
|
break;
|
|
case libcamera::formats::NV24:
|
|
horzSubSample_ = 1;
|
|
vertSubSample_ = 1;
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_UV");
|
|
fragmentShaderFile_ = ":YUV_2_planes.frag";
|
|
break;
|
|
case libcamera::formats::NV42:
|
|
horzSubSample_ = 1;
|
|
vertSubSample_ = 1;
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_VU");
|
|
fragmentShaderFile_ = ":YUV_2_planes.frag";
|
|
break;
|
|
case libcamera::formats::YUV420:
|
|
horzSubSample_ = 2;
|
|
vertSubSample_ = 2;
|
|
fragmentShaderFile_ = ":YUV_3_planes.frag";
|
|
break;
|
|
case libcamera::formats::YVU420:
|
|
horzSubSample_ = 2;
|
|
vertSubSample_ = 2;
|
|
fragmentShaderFile_ = ":YUV_3_planes.frag";
|
|
break;
|
|
case libcamera::formats::UYVY:
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY");
|
|
fragmentShaderFile_ = ":YUV_packed.frag";
|
|
break;
|
|
case libcamera::formats::VYUY:
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY");
|
|
fragmentShaderFile_ = ":YUV_packed.frag";
|
|
break;
|
|
case libcamera::formats::YUYV:
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV");
|
|
fragmentShaderFile_ = ":YUV_packed.frag";
|
|
break;
|
|
case libcamera::formats::YVYU:
|
|
fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU");
|
|
fragmentShaderFile_ = ":YUV_packed.frag";
|
|
break;
|
|
case libcamera::formats::ABGR8888:
|
|
fragmentShaderDefines_.append("#define RGB_PATTERN rgb");
|
|
fragmentShaderFile_ = ":RGB.frag";
|
|
break;
|
|
case libcamera::formats::ARGB8888:
|
|
fragmentShaderDefines_.append("#define RGB_PATTERN bgr");
|
|
fragmentShaderFile_ = ":RGB.frag";
|
|
break;
|
|
case libcamera::formats::BGRA8888:
|
|
fragmentShaderDefines_.append("#define RGB_PATTERN gba");
|
|
fragmentShaderFile_ = ":RGB.frag";
|
|
break;
|
|
case libcamera::formats::RGBA8888:
|
|
fragmentShaderDefines_.append("#define RGB_PATTERN abg");
|
|
fragmentShaderFile_ = ":RGB.frag";
|
|
break;
|
|
case libcamera::formats::BGR888:
|
|
fragmentShaderDefines_.append("#define RGB_PATTERN rgb");
|
|
fragmentShaderFile_ = ":RGB.frag";
|
|
break;
|
|
case libcamera::formats::RGB888:
|
|
fragmentShaderDefines_.append("#define RGB_PATTERN bgr");
|
|
fragmentShaderFile_ = ":RGB.frag";
|
|
break;
|
|
default:
|
|
ret = false;
|
|
qWarning() << "[ViewFinderGL]:"
|
|
<< "format not supported.";
|
|
break;
|
|
};
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool ViewFinderGL::createVertexShader()
|
|
{
|
|
/* Create Vertex Shader */
|
|
vertexShader_ = std::make_unique<QOpenGLShader>(QOpenGLShader::Vertex, this);
|
|
|
|
/* Compile the vertex shader */
|
|
if (!vertexShader_->compileSourceFile(":identity.vert")) {
|
|
qWarning() << "[ViewFinderGL]:" << vertexShader_->log();
|
|
return false;
|
|
}
|
|
|
|
shaderProgram_.addShader(vertexShader_.get());
|
|
return true;
|
|
}
|
|
|
|
bool ViewFinderGL::createFragmentShader()
|
|
{
|
|
int attributeVertex;
|
|
int attributeTexture;
|
|
|
|
/*
|
|
* Create the fragment shader, compile it, and add it to the shader
|
|
* program. The #define macros stored in fragmentShaderDefines_, if
|
|
* any, are prepended to the source code.
|
|
*/
|
|
fragmentShader_ = std::make_unique<QOpenGLShader>(QOpenGLShader::Fragment, this);
|
|
|
|
QFile file(fragmentShaderFile_);
|
|
if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
|
|
qWarning() << "Shader" << fragmentShaderFile_ << "not found";
|
|
return false;
|
|
}
|
|
|
|
QString defines = fragmentShaderDefines_.join('\n') + "\n";
|
|
QByteArray src = file.readAll();
|
|
src.prepend(defines.toUtf8());
|
|
|
|
if (!fragmentShader_->compileSourceCode(src)) {
|
|
qWarning() << "[ViewFinderGL]:" << fragmentShader_->log();
|
|
return false;
|
|
}
|
|
|
|
shaderProgram_.addShader(fragmentShader_.get());
|
|
|
|
/* Link shader pipeline */
|
|
if (!shaderProgram_.link()) {
|
|
qWarning() << "[ViewFinderGL]:" << shaderProgram_.log();
|
|
close();
|
|
}
|
|
|
|
/* Bind shader pipeline for use */
|
|
if (!shaderProgram_.bind()) {
|
|
qWarning() << "[ViewFinderGL]:" << shaderProgram_.log();
|
|
close();
|
|
}
|
|
|
|
attributeVertex = shaderProgram_.attributeLocation("vertexIn");
|
|
attributeTexture = shaderProgram_.attributeLocation("textureIn");
|
|
|
|
shaderProgram_.enableAttributeArray(attributeVertex);
|
|
shaderProgram_.setAttributeBuffer(attributeVertex,
|
|
GL_FLOAT,
|
|
0,
|
|
2,
|
|
2 * sizeof(GLfloat));
|
|
|
|
shaderProgram_.enableAttributeArray(attributeTexture);
|
|
shaderProgram_.setAttributeBuffer(attributeTexture,
|
|
GL_FLOAT,
|
|
8 * sizeof(GLfloat),
|
|
2,
|
|
2 * sizeof(GLfloat));
|
|
|
|
textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
|
|
textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
|
|
textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
|
|
textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx");
|
|
|
|
/* Create the textures. */
|
|
for (std::unique_ptr<QOpenGLTexture> &texture : textures_) {
|
|
if (texture)
|
|
continue;
|
|
|
|
texture = std::make_unique<QOpenGLTexture>(QOpenGLTexture::Target2D);
|
|
texture->create();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void ViewFinderGL::configureTexture(QOpenGLTexture &texture)
|
|
{
|
|
glBindTexture(GL_TEXTURE_2D, texture.textureId());
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
|
|
}
|
|
|
|
void ViewFinderGL::removeShader()
|
|
{
|
|
if (shaderProgram_.isLinked()) {
|
|
shaderProgram_.release();
|
|
shaderProgram_.removeAllShaders();
|
|
}
|
|
}
|
|
|
|
void ViewFinderGL::initializeGL()
|
|
{
|
|
initializeOpenGLFunctions();
|
|
glEnable(GL_TEXTURE_2D);
|
|
glDisable(GL_DEPTH_TEST);
|
|
|
|
static const GLfloat coordinates[2][4][2]{
|
|
{
|
|
/* Vertex coordinates */
|
|
{ -1.0f, -1.0f },
|
|
{ -1.0f, +1.0f },
|
|
{ +1.0f, +1.0f },
|
|
{ +1.0f, -1.0f },
|
|
},
|
|
{
|
|
/* Texture coordinates */
|
|
{ 0.0f, 1.0f },
|
|
{ 0.0f, 0.0f },
|
|
{ 1.0f, 0.0f },
|
|
{ 1.0f, 1.0f },
|
|
},
|
|
};
|
|
|
|
vertexBuffer_.create();
|
|
vertexBuffer_.bind();
|
|
vertexBuffer_.allocate(coordinates, sizeof(coordinates));
|
|
|
|
/* Create Vertex Shader */
|
|
if (!createVertexShader())
|
|
qWarning() << "[ViewFinderGL]: create vertex shader failed.";
|
|
|
|
glClearColor(1.0f, 1.0f, 1.0f, 0.0f);
|
|
}
|
|
|
|
void ViewFinderGL::doRender()
|
|
{
|
|
switch (format_) {
|
|
case libcamera::formats::NV12:
|
|
case libcamera::formats::NV21:
|
|
case libcamera::formats::NV16:
|
|
case libcamera::formats::NV61:
|
|
case libcamera::formats::NV24:
|
|
case libcamera::formats::NV42:
|
|
/* Activate texture Y */
|
|
glActiveTexture(GL_TEXTURE0);
|
|
configureTexture(*textures_[0]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width(),
|
|
size_.height(),
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_);
|
|
shaderProgram_.setUniformValue(textureUniformY_, 0);
|
|
|
|
/* Activate texture UV/VU */
|
|
glActiveTexture(GL_TEXTURE1);
|
|
configureTexture(*textures_[1]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RG,
|
|
size_.width() / horzSubSample_,
|
|
size_.height() / vertSubSample_,
|
|
0,
|
|
GL_RG,
|
|
GL_UNSIGNED_BYTE,
|
|
data_ + size_.width() * size_.height());
|
|
shaderProgram_.setUniformValue(textureUniformU_, 1);
|
|
break;
|
|
|
|
case libcamera::formats::YUV420:
|
|
/* Activate texture Y */
|
|
glActiveTexture(GL_TEXTURE0);
|
|
configureTexture(*textures_[0]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width(),
|
|
size_.height(),
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_);
|
|
shaderProgram_.setUniformValue(textureUniformY_, 0);
|
|
|
|
/* Activate texture U */
|
|
glActiveTexture(GL_TEXTURE1);
|
|
configureTexture(*textures_[1]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width() / horzSubSample_,
|
|
size_.height() / vertSubSample_,
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_ + size_.width() * size_.height());
|
|
shaderProgram_.setUniformValue(textureUniformU_, 1);
|
|
|
|
/* Activate texture V */
|
|
glActiveTexture(GL_TEXTURE2);
|
|
configureTexture(*textures_[2]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width() / horzSubSample_,
|
|
size_.height() / vertSubSample_,
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_ + size_.width() * size_.height() * 5 / 4);
|
|
shaderProgram_.setUniformValue(textureUniformV_, 2);
|
|
break;
|
|
|
|
case libcamera::formats::YVU420:
|
|
/* Activate texture Y */
|
|
glActiveTexture(GL_TEXTURE0);
|
|
configureTexture(*textures_[0]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width(),
|
|
size_.height(),
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_);
|
|
shaderProgram_.setUniformValue(textureUniformY_, 0);
|
|
|
|
/* Activate texture V */
|
|
glActiveTexture(GL_TEXTURE2);
|
|
configureTexture(*textures_[2]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width() / horzSubSample_,
|
|
size_.height() / vertSubSample_,
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_ + size_.width() * size_.height());
|
|
shaderProgram_.setUniformValue(textureUniformV_, 2);
|
|
|
|
/* Activate texture U */
|
|
glActiveTexture(GL_TEXTURE1);
|
|
configureTexture(*textures_[1]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RED,
|
|
size_.width() / horzSubSample_,
|
|
size_.height() / vertSubSample_,
|
|
0,
|
|
GL_RED,
|
|
GL_UNSIGNED_BYTE,
|
|
data_ + size_.width() * size_.height() * 5 / 4);
|
|
shaderProgram_.setUniformValue(textureUniformU_, 1);
|
|
break;
|
|
|
|
case libcamera::formats::UYVY:
|
|
case libcamera::formats::VYUY:
|
|
case libcamera::formats::YUYV:
|
|
case libcamera::formats::YVYU:
|
|
/*
|
|
* Packed YUV formats are stored in a RGBA texture to match the
|
|
* OpenGL texel size with the 4 bytes repeating pattern in YUV.
|
|
* The texture width is thus half of the image with.
|
|
*/
|
|
glActiveTexture(GL_TEXTURE0);
|
|
configureTexture(*textures_[0]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RGBA,
|
|
size_.width() / 2,
|
|
size_.height(),
|
|
0,
|
|
GL_RGBA,
|
|
GL_UNSIGNED_BYTE,
|
|
data_);
|
|
shaderProgram_.setUniformValue(textureUniformY_, 0);
|
|
|
|
/*
|
|
* The shader needs the step between two texture pixels in the
|
|
* horizontal direction, expressed in texture coordinate units
|
|
* ([0, 1]). There are exactly width - 1 steps between the
|
|
* leftmost and rightmost texels.
|
|
*/
|
|
shaderProgram_.setUniformValue(textureUniformStepX_,
|
|
1.0f / (size_.width() / 2 - 1));
|
|
break;
|
|
|
|
case libcamera::formats::ABGR8888:
|
|
case libcamera::formats::ARGB8888:
|
|
case libcamera::formats::BGRA8888:
|
|
case libcamera::formats::RGBA8888:
|
|
glActiveTexture(GL_TEXTURE0);
|
|
configureTexture(*textures_[0]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RGBA,
|
|
size_.width(),
|
|
size_.height(),
|
|
0,
|
|
GL_RGBA,
|
|
GL_UNSIGNED_BYTE,
|
|
data_);
|
|
shaderProgram_.setUniformValue(textureUniformY_, 0);
|
|
break;
|
|
|
|
case libcamera::formats::BGR888:
|
|
case libcamera::formats::RGB888:
|
|
glActiveTexture(GL_TEXTURE0);
|
|
configureTexture(*textures_[0]);
|
|
glTexImage2D(GL_TEXTURE_2D,
|
|
0,
|
|
GL_RGB,
|
|
size_.width(),
|
|
size_.height(),
|
|
0,
|
|
GL_RGB,
|
|
GL_UNSIGNED_BYTE,
|
|
data_);
|
|
shaderProgram_.setUniformValue(textureUniformY_, 0);
|
|
break;
|
|
|
|
default:
|
|
break;
|
|
};
|
|
}
|
|
|
|
void ViewFinderGL::paintGL()
|
|
{
|
|
if (!fragmentShader_)
|
|
if (!createFragmentShader()) {
|
|
qWarning() << "[ViewFinderGL]:"
|
|
<< "create fragment shader failed.";
|
|
}
|
|
|
|
if (data_) {
|
|
glClearColor(0.0, 0.0, 0.0, 1.0);
|
|
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
|
|
|
|
doRender();
|
|
glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
|
|
}
|
|
}
|
|
|
|
void ViewFinderGL::resizeGL(int w, int h)
|
|
{
|
|
glViewport(0, 0, w, h);
|
|
}
|
|
|
|
QSize ViewFinderGL::sizeHint() const
|
|
{
|
|
return size_.isValid() ? size_ : QSize(640, 480);
|
|
}
|