x86平台基于Qt+opengl优化ffmpeg软解码1080P视频渲染效率

一般的在arm嵌入式平台，大多数板子都要硬解码硬件渲染的框架，使用即可。

在x86下比较麻烦了。

优化的思路一共有以下几个方面，

1. 软解码变成硬解码

2. 将YUV转QImage的操作转移到GPU

3. QWidget渲染QImage变成opengGL渲染AVFrame

这三点优化来说2与3是优化的效率是非常显著的。

1的优化效果往往需要将硬解码的数据copy至CPU再使用2-3的优化。

这样一来，解码效率提升了，但是数据copy时候CPU使用率会上升。如果两者抵消后CPU使用率还是上升那就得不偿失。如果能实现硬解码的数据不经过CPU直接打到GPU进行渲染，那就是最完美的方案。这个在x86下需要研究opengl渲染硬件类型数据，难度未知，理论如果用的是比较新的框架，资料会多一些。

本文主要是基于2-3的优化，在qt5.1下面基于opengl实现了这个方案，在多路1080P的使用场景下CPU使用率下降非常明显。

#include "opengl_yuv_shader.h"
#include <QDebug>
#include <iostream>
#include <GL/gl.h>
#include <QGLShader>opengl_yuv_shader::opengl_yuv_shader(QWidget *parent) : QGLWidget(parent), useVBO(false),vboId(0),yuv420p_shaderProgram(0),yuvj422p_shaderProgram(0)
{textures[0]=0;textures[1]=0;textures[2]=0;av_frame = nullptr;connect(this,SIGNAL(render_frame()),this,SLOT(slot_render_frame()),Qt::QueuedConnection);//5 lu 60% cpu
}opengl_yuv_shader::~opengl_yuv_shader() {makeCurrent();glDeleteTextures(3, textures);if (yuv420p_shaderProgram) {glDeleteProgram(yuv420p_shaderProgram);}if (yuvj422p_shaderProgram) {glDeleteProgram(yuvj422p_shaderProgram);}doneCurrent();
}void opengl_yuv_shader::initTextures()
{glGenTextures(3, textures);for (int i = 0; i < 3; ++i) {glBindTexture(GL_TEXTURE_2D, textures[i]);glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);glBindTexture(GL_TEXTURE_2D, 0);}
}void opengl_yuv_shader::initShaders()
{QGLShader *vshader = new QGLShader(QGLShader::Vertex, this);const char *vsrc ="attribute vec4 vertex;\n""attribute vec2 texCoord;\n""varying vec2 texc;\n""void main(void)\n""{\n""    gl_Position = vertex;\n""    texc = texCoord;\n""}\n";vshader->compileSourceCode(vsrc);//编译顶点着色器代码QGLShader *fshader = new QGLShader(QGLShader::Fragment, this);//vec4(1.0,0,0,1.0);const char *fsrc ="uniform sampler2D texture;\n""varying vec2 texc;\n""void main(void)\n""{\n""    gl_FragColor =  texture2D(texture,texc);\n""}\n";//本方案的核心点在于这个片段着色器，在GPU上完成YUV转RGB的浮点运算。//由于测试的摄像机是基于YUV J420P转换的所以算法上与YUV420P略有差别。// 实际使用需要根据具体的AVFrame格式，进行转换。可初始化多个SHADER管理器、// 渲染时，根据像素格式选择shader渲染const char* fragmentShaderSource = R"(varying vec2 texc;uniform sampler2D textureY;uniform sampler2D textureU;uniform sampler2D textureV;void main(){float y = texture2D(textureY, texc).r;float u = texture2D(textureU, texc).r;float v = texture2D(textureV, texc).r;float r = y + 1.402 * (v - 0.5);float g = y - 0.344136 * (u - 0.5) - 0.714136 * (v - 0.5);float b = y + 1.772 * (u - 0.5);// 确保 RGB 值在 0-1 范围内r = clamp(r, 0.0, 1.0);g = clamp(g, 0.0, 1.0);b = clamp(b, 0.0, 1.0);gl_FragColor = vec4(r, g, b, 1.0);})";fshader->compileSourceCode(fragmentShaderSource); //编译纹理着色器代码program.addShader(vshader);//添加顶点着色器program.addShader(fshader);//添加纹理碎片着色器program.bindAttributeLocation("vertex", 0);//绑定顶点属性位置program.bindAttributeLocation("texCoord", 1);//绑定纹理属性位置// 链接着色器管道if (!program.link()){close();qDebug()<<"program.link() error"<<endl;}// 绑定着色器管道if (!program.bind()){close();qDebug()<<"program.bind() error"<<endl;}
}void opengl_yuv_shader::initializeGL()
{initializeOpenGLFunctions();glClearColor(0.0f, 0.0f, 0.0f, 1.0f);glEnable(GL_TEXTURE_2D);initTextures();initShaders();
//    glDisable(GL_DEPTH_TEST);
//    glDisable(GL_CULL_FACE);
//    glDisable(GL_BLEND);const GLubyte* renderer = glGetString(GL_RENDERER);const GLubyte* vendor = glGetString(GL_VENDOR);const GLubyte* version = glGetString(GL_VERSION);const GLubyte* glslVersion = glGetString(GL_SHADING_LANGUAGE_VERSION);std::cout << "Renderer: " << renderer<<std::endl;std::cout << "Vendor: " << vendor<<std::endl;std::cout << "OpenGL Version: " << version<<std::endl;std::cout << "GLSL Version: " << glslVersion<<std::endl;texCoords.append(QVector2D(0, 1)); //左上texCoords.append(QVector2D(1, 1)); //右上texCoords.append(QVector2D(0, 0)); //左下texCoords.append(QVector2D(1, 0)); //右下//顶点坐标vertices.append(QVector3D(-1, -1, 1));//左下vertices.append(QVector3D(1, -1, 1)); //右下vertices.append(QVector3D(-1, 1, 1)); //左上vertices.append(QVector3D(1, 1, 1));  //右上
}void opengl_yuv_shader::resizeGL(int w, int h)
{qDebug() << "Oopengl_yuv_shader::resizeGL w=" << w<<endl;glViewport(0, 0, w, h);glMatrixMode(GL_PROJECTION);glLoadIdentity();glOrtho(-1.0, 1.0, -1.0, 1.0, -1.0, 1.0);glMatrixMode(GL_MODELVIEW);
}void opengl_yuv_shader::paintGL()
{glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);render_lock.lock();if (!av_frame) {render_lock.unlock();return;}glEnable(GL_TEXTURE_2D);program.enableAttributeArray(0);//启用顶点属性0，也就是渲染平面的顶点坐标program.enableAttributeArray(1);//启用顶点属性1，也就是渲染平面的纹理坐标//纹理坐标的和顶点的对应关系完成渲染program.setAttributeArray(0, vertices.constData() );program.setAttributeArray(1, texCoords.constData()  );if(av_frame->format == AV_PIX_FMT_YUV420P || av_frame->format == AV_PIX_FMT_YUVJ420P  ){if (av_frame&&av_frame->data[0]) {glActiveTexture(GL_TEXTURE0);glBindTexture(GL_TEXTURE_2D, textures[0]);glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, av_frame->width, av_frame->height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, av_frame->data[0]);glActiveTexture(GL_TEXTURE1);glBindTexture(GL_TEXTURE_2D, textures[1]);glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, av_frame->width/2, av_frame->height/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, av_frame->data[1]);glActiveTexture(GL_TEXTURE2);glBindTexture(GL_TEXTURE_2D, textures[2]);glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, av_frame->width/2, av_frame->height/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, av_frame->data[2]);program.setUniformValue("textureY", 0);program.setUniformValue("textureU", 1);program.setUniformValue("textureV", 2);}}render_lock.unlock();// 绘制glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);}void opengl_yuv_shader::set_yuv_frame(AVFrame *frame)
{// 1. 如果 av_frame 已经存在，先释放它render_lock.lock();if (av_frame) {av_frame_free(&av_frame);av_frame = nullptr;}// 2. 深拷贝 AVFrameav_frame = av_frame_clone(frame);if (!av_frame) {av_log(NULL, AV_LOG_ERROR, "Failed to clone frame\n");render_lock.unlock();return;}render_lock.unlock();emit render_frame();}void opengl_yuv_shader::slot_render_frame()
{update();
}