SDL2绘制ffmpeg解析的mp4文件

---

本项目采用生产者消费者模型，生产者线程：使用ffmpeg将mp4格式数据解析为yuv的帧，消费者线程：利用sdl2将解析的yuv的帧进行消费，绘制到屏幕上。未完成的部分：1.解析音频数据，并与视频数据同步。2.增加界面：暂停，播放按钮，支持视频前进和后退。
学习音视频的参考资料与项目：
playdemo_github
雷神的csdn博客

---

1.FFMPEG利用命令行将mp4转yuv420

ffmpeg -i input.mp4 -c:v rawvideo -pix_fmt yuv420p output.yuv

2.ffmpeg将mp4解析为yuv数据

2.1 核心api:

• av_read_frame:读取一帧数据
• avcodec_send_packet:将数据包发送给解码器
• avcodec_receive_frame:将数据包从解码器中取
• sws_scale:格式转换，将解码后的帧数据转为yuv数据，存储在data[0],data[1],data[2]中
  

void readFrame()
{AVPacket* avPacket = av_packet_alloc();AVFrame* frame = av_frame_alloc();FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data","wb+") ;while (av_read_frame(formatContext, avPacket) >= 0 && fp){if (avPacket->stream_index == videoStreamIndex){if (avcodec_send_packet(codecContext, avPacket) < 0) {std::cerr << "发送数据包到解码器失败" << std::endl;break;}/*解码*/int ret = avcodec_receive_frame(codecContext, frame);printf("ret:%d\n", ret);if (ret >= 0){ret = sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, yuvFrame->data, yuvFrame->linesize);printf("sws_scale ret=%d\n", ret);std::lock_guard<std::mutex>lck(mtx);isFinished = false;memcpy(yuvBuf, yuvFrame->data[0], yuvFrame->width * yuvFrame->height);memcpy(yuvBuf + yuvFrame->width * yuvFrame->height, yuvFrame->data[1], yuvFrame->width * yuvFrame->height / 4);memcpy(yuvBuf + yuvFrame->width * yuvFrame->height*5/4, yuvFrame->data[2], yuvFrame->width * yuvFrame->height / 4);isFinished = true;condvar.notify_one();//保存y分量//fwrite(yuvFrame->data[0], 1, yuvFrame->width * yuvFrame->height, fp);//保存uv分量//fwrite(yuvFrame->data[1], 1, yuvFrame->width * yuvFrame->height/4, fp);//fwrite(yuvFrame->data[2], 1, yuvFrame->width * yuvFrame->height / 4, fp);}}}fclose(fp);av_frame_unref(yuvFrame);av_packet_free(&avPacket);av_frame_unref(frame);
}

3.SDL2进行yuv绘制到屏幕

3.1 核心api

• SDL_Init
• SDL_CreateWindow
• SDL_CreateRenderer
• SDL_CreateTexture
• SDL_UpdateTexture
• SDL_RenderCopy
• SDL_RenderPresent
• SDL_Delay:控制帧率
  

int sdl_display()
{if (SDL_Init(SDL_INIT_VIDEO)) {printf("sdl init failed\n");return -1;}SDL_Window* window = SDL_CreateWindow("sdl_demo", 200, 200, codecContext->width, codecContext->height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);if (!window) {SDL_Quit();return -1;}SDL_Renderer* renderer = SDL_CreateRenderer(window, -1, 0);if (!renderer){SDL_DestroyWindow(window);SDL_Quit();return -1;}SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);SDL_RenderClear(renderer);Uint32 pixformat = SDL_PIXELFORMAT_IYUV;SDL_Texture* sdlTexture = SDL_CreateTexture(renderer, pixformat, SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);//FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data", "rb+");while (1) {//int ret = fread(yuvBuf, 1, yuvlen, fp);//if (ret <= 0) {//	break;//}std::unique_lock<std::mutex>lck(mtx);if (condvar.wait_for(lck, std::chrono::seconds(1), [] {return isFinished;})){isFinished = false;SDL_UpdateTexture(sdlTexture, NULL, yuvBuf, codecContext->width);SDL_RenderCopy(renderer, sdlTexture, NULL, NULL);SDL_RenderPresent(renderer);//控制帧率25fpsSDL_Delay(40);}else {printf("sdl thread exit!\n");break;}}SDL_Quit();return 0;
}

4.完整代码

-使用两个线程，生产者消费者模型

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <thread>
#include <fstream>#include <mutex>
#include <condition_variable>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libswscale/swscale.h>#include <SDL.h>
}
#undef main
#pragma warning(disable:4996)
AVFormatContext* formatContext = nullptr;
AVCodecContext* codecContext = nullptr;
SwsContext* swsContext = nullptr;
int videoStreamIndex = -1;
AVFrame* yuvFrame;
unsigned char* yuvBuf;
bool isReady = false;
bool isFinished = false;std::mutex mtx;
std::condition_variable condvar;
void readFrame()
{AVPacket* avPacket = av_packet_alloc();AVFrame* frame = av_frame_alloc();FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data","wb+") ;while (av_read_frame(formatContext, avPacket) >= 0 && fp){if (avPacket->stream_index == videoStreamIndex){if (avcodec_send_packet(codecContext, avPacket) < 0) {std::cerr << "发送数据包到解码器失败" << std::endl;break;}/*解码*/int ret = avcodec_receive_frame(codecContext, frame);printf("ret:%d\n", ret);if (ret >= 0){ret = sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, yuvFrame->data, yuvFrame->linesize);printf("sws_scale ret=%d\n", ret);std::lock_guard<std::mutex>lck(mtx);isFinished = false;memcpy(yuvBuf, yuvFrame->data[0], yuvFrame->width * yuvFrame->height);memcpy(yuvBuf + yuvFrame->width * yuvFrame->height, yuvFrame->data[1], yuvFrame->width * yuvFrame->height / 4);memcpy(yuvBuf + yuvFrame->width * yuvFrame->height*5/4, yuvFrame->data[2], yuvFrame->width * yuvFrame->height / 4);isFinished = true;condvar.notify_one();//保存y分量//fwrite(yuvFrame->data[0], 1, yuvFrame->width * yuvFrame->height, fp);//保存uv分量//fwrite(yuvFrame->data[1], 1, yuvFrame->width * yuvFrame->height/4, fp);//fwrite(yuvFrame->data[2], 1, yuvFrame->width * yuvFrame->height / 4, fp);}}}fclose(fp);av_frame_unref(yuvFrame);av_packet_free(&avPacket);av_frame_unref(frame);
}int sdl_display()
{if (SDL_Init(SDL_INIT_VIDEO)) {printf("sdl init failed\n");return -1;}SDL_Window* window = SDL_CreateWindow("sdl_demo", 200, 200, codecContext->width, codecContext->height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);if (!window) {SDL_Quit();return -1;}SDL_Renderer* renderer = SDL_CreateRenderer(window, -1, 0);if (!renderer){SDL_DestroyWindow(window);SDL_Quit();return -1;}SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);SDL_RenderClear(renderer);Uint32 pixformat = SDL_PIXELFORMAT_IYUV;SDL_Texture* sdlTexture = SDL_CreateTexture(renderer, pixformat, SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);//FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data", "rb+");while (1) {//int ret = fread(yuvBuf, 1, yuvlen, fp);//if (ret <= 0) {//	break;//}std::unique_lock<std::mutex>lck(mtx);if (condvar.wait_for(lck, std::chrono::seconds(1), [] {return isFinished;})){isFinished = false;SDL_UpdateTexture(sdlTexture, NULL, yuvBuf, codecContext->width);SDL_RenderCopy(renderer, sdlTexture, NULL, NULL);SDL_RenderPresent(renderer);//控制帧率25fpsSDL_Delay(40);}else {printf("sdl thread exit!\n");break;}}SDL_Quit();return 0;
}/*ffmpeg -i input.mp4 -c:v rawvideo -pix_fmt yuv420p output.yuv*/
int main(int argc, char* argv[])
{/*if (argc != 2) {std::cerr << "文件名未指定" << std::endl;return -1;}*/std::string filename = "F:/VS_Project/ffmpeg_demo/1.mkv";if (avformat_open_input(&formatContext, filename.c_str(), nullptr, nullptr) != 0){std::cerr << "无法打开文件" << std::endl;return -1;}if (avformat_find_stream_info(formatContext, nullptr) < 0) {std::cerr << "无法找到视频流" << std::endl;return -1;}for (int i = 0; i < formatContext->nb_streams; i++){enum AVMediaType type = AVMEDIA_TYPE_VIDEO;AVStream* st = formatContext->streams[i];AVCodecParameters* codecpar = st->codecpar;if (codecpar->codec_type == type){videoStreamIndex = i;const AVCodec* codec = avcodec_find_decoder(codecpar->codec_id);codecContext = avcodec_alloc_context3(codec);avcodec_parameters_to_context(codecContext, codecpar);avcodec_open2(codecContext, codec, nullptr);swsContext = sws_getContext(codecContext->width, codecContext->height, codecContext->pix_fmt,codecContext->width, codecContext->height, AV_PIX_FMT_YUV420P,SWS_BILINEAR, nullptr, nullptr, nullptr);std::cout << "w:" << codecpar->width << std::endl;std::cout << "h:" << codecpar->height << std::endl;}}yuvFrame = av_frame_alloc();yuvFrame->width = codecContext->width;yuvFrame->height = codecContext->height;yuvFrame->format = AV_PIX_FMT_YUV420P;int yuvlen = codecContext->width * codecContext->height * 3 / 2;yuvBuf = new unsigned char[yuvlen];int ret = av_frame_get_buffer(yuvFrame, 0);if (ret < 0) {printf("分配缓冲区失败\n");return -1;}//sdl_init();std::thread th1(readFrame);std::thread th2(sdl_display);th1.join();th2.join();delete[]yuvBuf;return 0;
}

5.效果展示

6.SDL2事件响应补充

6.1 处理方式-01

起一个refresh_video线程，用于产生一个自定义更新video的事件：REFRESH_EVENT，并且每40ms更新一次。while循环中持续等待事件的到来：SDL_WaitEvent，当收到事件后更新一帧画面。同时当监测到窗口改变的事件，SDL_GetWindowSize对窗口进行调整。

int refresh_video(void *opaque){thread_exit=0;while (thread_exit==0) {SDL_Event event;event.type = REFRESH_EVENT;SDL_PushEvent(&event);SDL_Delay(40);}thread_exit=0;//BreakSDL_Event event;event.type = BREAK_EVENT;SDL_PushEvent(&event);return 0;
}
SDL_Thread *refresh_thread = SDL_CreateThread(refresh_video,NULL,NULL);SDL_Event event;while(1){//WaitSDL_WaitEvent(&event);if(event.type==REFRESH_EVENT){if (fread(buffer, 1, pixel_w*pixel_h*bpp/8, fp) != pixel_w*pixel_h*bpp/8){// Loopfseek(fp, 0, SEEK_SET);fread(buffer, 1, pixel_w*pixel_h*bpp/8, fp);}SDL_UpdateTexture( sdlTexture, NULL, buffer, pixel_w);  //FIX: If window is resizesdlRect.x = 0;  sdlRect.y = 0;  sdlRect.w = screen_w;  sdlRect.h = screen_h;  SDL_RenderClear( sdlRenderer );   SDL_RenderCopy( sdlRenderer, sdlTexture, NULL, &sdlRect);  SDL_RenderPresent( sdlRenderer );  }else if(event.type==SDL_WINDOWEVENT){//If ResizeSDL_GetWindowSize(screen,&screen_w,&screen_h);}else if(event.type==SDL_QUIT){thread_exit=1;}else if(event.type==BREAK_EVENT){break;}}SDL_Quit();

6.2 处理方式-02

起一个事件循环线程，SDL_PeepEvents 从事件队列中取出一个事件，然后更新事件队列，并进行绘制操作。

//播放控制循环
void VideoCtl::LoopThread(VideoState *cur_stream)
{SDL_Event event;double incr, pos, frac;m_bPlayLoop = true;while (m_bPlayLoop){double x;refresh_loop_wait_event(cur_stream, &event);switch (event.type) {case SDL_KEYDOWN:switch (event.key.keysym.sym) {case SDLK_s: // S: Step to next framestep_to_next_frame(cur_stream);break;case SDLK_a:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);break;case SDLK_v:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);break;case SDLK_c:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);break;case SDLK_t:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);break;default:break;}break;case SDL_WINDOWEVENT://窗口大小改变事件qDebug()<<"SDL_WINDOWEVENT  "<<endl;switch (event.window.event) {case SDL_WINDOWEVENT_RESIZED:screen_width = cur_stream->width = event.window.data1;screen_height = cur_stream->height = event.window.data2;case SDL_WINDOWEVENT_EXPOSED:cur_stream->force_refresh = 1;}break;case SDL_QUIT:case FF_QUIT_EVENT:do_exit(cur_stream);break;default:break;}}do_exit(m_CurStream);//m_CurStream = nullptr;}