FFmpeg入门：最简单的音视频播放器

FFmpeg入门：最简单的音视频播放器

前两章，我们已经了解了分别如何构建一个简单和音频播放器和视频播放器。
FFmpeg入门：最简单的音频播放器
FFmpeg入门：最简单的视频播放器

本章我们将结合上述两章的知识，看看如何融合成一个完整的音视频播放器，跟上我的节奏，本章将是咱们后续完成一个完整的音视频播放器的起点。

整体流程图

话不多说，先上图

这个图似乎有点复杂了，这里我会分别将每个模块拿出来讲述，方便大家一步一步分析整个流程。

第一步：初始化

我们首先关注整个流程图的最上面一部分，这部分其实和之前的流程一样，主要就是将做一些前置的初始化工作：

1：打开文件，获取文件上下文
2：找到对应的音频/视频流，获取到Codec上下文
3：打开解码器
4：分配输出空间缓存，用于后续存储解码的输出数据
5：音频/视频帧的格式转化上下文
6：初始化SDL组件，主要是视频的播放窗口和音频播放器

代码（省略了部分校验和参数初始化，方便阅读，原码见文章末尾）：

	/** 初始化函数 */init_video_state(&video_state);audio_param = video_state->audioParam;video_param = video_state->videoParam;avformat_network_init();// 1. 打开视频文件，获取格式上下文if(avformat_open_input(&video_state->formatCtx, argv[1], NULL, NULL)!=0){printf("Couldn't open input stream.\n");return -1;}// 2. 对文件探测流信息if(avformat_find_stream_info(video_state->formatCtx, NULL) < 0){printf("Couldn't find stream information.\n");return -1;}// 3. 找到对应的 音频流/视频流 索引video_state->audioStream=-1;video_state->videoStream=-1;for(int i=0; i < video_state->formatCtx->nb_streams; i++) {if(video_state->formatCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_AUDIO){video_state->audioStream=i;}if (video_state->formatCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_VIDEO) {video_state->videoStream=i;}}// 4. 将 音频流/视频流 编码参数写入上下文AVCodecParameters* aCodecParam = video_state->formatCtx->streams[video_state->audioStream]->codecpar;avcodec_parameters_to_context(video_state->aCodecCtx, aCodecParam);AVCodecParameters* vCodecParam = video_state->formatCtx->streams[video_state->videoStream]->codecpar;avcodec_parameters_to_context(video_state->vCodecCtx, vCodecParam);// 5. 查找流的编码器video_state->aCodec = avcodec_find_decoder(video_state->aCodecCtx->codec_id);video_state->vCodec = avcodec_find_decoder(video_state->vCodecCtx->codec_id);// 6. 打开流的编解码器if(avcodec_open2(video_state->aCodecCtx, video_state->aCodec, NULL)<0){printf("Could not open audio codec.\n");return -1;}if(avcodec_open2(video_state->vCodecCtx, video_state->vCodec, NULL)<0){printf("Could not open video codec.\n");return -1;}/** 音频输出信息构建 */audio_output_set(video_state);/** 视频输出信息构建 */video_output_set(video_state);//	SDL 初始化
#if USE_SDLif(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {printf( "Could not initialize SDL - %s\n", SDL_GetError());return -1;}/** 初始化音频SDL设备 */SDL_AudioSpec wanted_spec;wanted_spec.freq = audio_param->out_sample_rate;					// 采样率wanted_spec.format = AUDIO_S16SYS;									// 采样格式 16bitwanted_spec.channels = audio_param->out_channels;					// 通道数wanted_spec.silence = 0;wanted_spec.samples = audio_param->out_nb_samples;					// 单帧处理的采样点wanted_spec.callback = fill_audio;									// 回调函数wanted_spec.userdata = video_state->aCodecCtx;						// 回调函数的参数/** 初始化视频SDL设备 */SDL_Window*       window = NULL;SDL_Renderer*     renderer = NULL;SDL_Texture*      texture= NULL;/** 窗口 */window = SDL_CreateWindow("SDL2 window",SDL_WINDOWPOS_CENTERED,SDL_WINDOWPOS_CENTERED,video_state->vCodecCtx->width,video_state->vCodecCtx->height,SDL_WINDOW_SHOWN);/** 渲染 */renderer = SDL_CreateRenderer(window,-1,SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);/** 纹理 */texture = SDL_CreateTexture(renderer,SDL_PIXELFORMAT_YV12,SDL_TEXTUREACCESS_STREAMING,video_state->vCodecCtx->width,video_state->vCodecCtx->height);// 打开音频播放器if (SDL_OpenAudio(&wanted_spec, NULL)<0) {printf("can't open audio.\n");return -1;}#endif// 音频上下文格式转换swr_alloc_set_opts2(&video_state->swrCtx,&audio_param->out_channel_layout,			// 输出layoutaudio_param->out_sample_fmt,				// 输出格式audio_param->out_sample_rate,				// 输出采样率&video_state->aCodecCtx->ch_layout,			// 输入layoutvideo_state->aCodecCtx->sample_fmt,			// 输入格式video_state->aCodecCtx->sample_rate,		// 输入采样率0, NULL);swr_init(video_state->swrCtx);// 视频上下文格式转换video_state->swsCtx = sws_getContext(video_state->vCodecCtx->width,			// src 宽video_state->vCodecCtx->height,		// src 高video_state->vCodecCtx->pix_fmt,		// src 格式video_param->width,					// dst 宽video_param->height,					// dst 高video_param->pix_fmt,					// dst 格式SWS_BILINEAR,NULL,NULL,NULL);// 开始播放SDL_PauseAudio(0);

第二步：packet队列写入

做完准备工作之后，我们就将源文件中的输入packet都取出来，放入到对应的音频packet队列和视频packet队列中，方便后续使用。然后然后分别启动音频解码进程和视频解码进程同时进行解码。

1. 读出packet，判断packet类型
2. 根据类型放入音频和视频packet队列
3. 创建解码进程

// 循环1: 从文件中读取packetwhile(av_read_frame(video_state->formatCtx, packet)>=0){/** 写入音频pkt队列 */if(packet->stream_index==video_state->audioStream){packet_queue_push(video_state->aQueue, packet);}/** 写入视频pkt队列 */if (packet->stream_index==video_state->videoStream) {packet_queue_push(video_state->vQueue, packet);}av_packet_unref(packet);SDL_PollEvent(&event);switch(event.type) {case SDL_QUIT:SDL_Quit();exit(0);break;default:break;}}printf("audio queue.size=%d\n", video_state->aQueue->size);// 创建一个线程并启动SDL_CreateThread(audio_thread, "audio_thread", video_state);SDL_CreateThread(video_thread, "video_thread", video_state);

第三步：音频解码+播放

接下来会分别讲一下音频和视频解码进程，这两个进程是同时开始的。

首先音频解码进程的步骤可以参考之前的音频播放器文章。我简单说一下步骤

1. 从音频packet队列中取出packet
2. 对packet进行解码得到frame
3. 按音频输出格式进行swr_convert转换得到输出值，并写入buffer。
4. SDL音频播放器通过回调函数从buffer不断读取数据播放。

代码如下：

/**音频线程*/
int audio_thread(void *arg) {/**1. 从packet_queue队列中取出packet2. 将packet进行解码3. 写入到sdl的缓冲区中*/ 	VideoState* video_state = (VideoState*) arg;AudioParam* audio_param = video_state->audioParam;PacketQueue* queue = video_state->aQueue;audio_param->index = 0;AVRational time_base = video_state->formatCtx->streams[video_state->audioStream]->time_base;int64_t av_start_time = av_gettime();								// 播放开始时间戳AVPacket 	packet;int 		ret;AVFrame* 	pFrame = av_frame_alloc();for(;;) {if (queue->size > 0) {packet_queue_pop(queue, &packet);// 将packet写入编解码器ret = avcodec_send_packet(video_state->aCodecCtx, &packet);// 获取解码后的帧while (!avcodec_receive_frame(video_state->aCodecCtx, pFrame)) {// 格式转化swr_convert(video_state->swrCtx, &audio_param->out_buffer, audio_param->out_buffer_size,(const uint8_t **)pFrame->data, pFrame->nb_samples);audio_param->index++;printf("第%d帧 | pts:%lld | 帧大小(采样点):%d | 实际播放点%.2fs | 预期播放点%.2fs\n",audio_param->index,packet.pts,packet.size,(double)(av_gettime() - av_start_time)/AV_TIME_BASE,pFrame->pts * av_q2d(time_base));#if USE_SDL// 设置读取的音频数据audio_info.audio_len = audio_param->out_buffer_size;audio_info.audio_pos = (Uint8 *) audio_param->out_buffer;// 等待SDL播放完成while(audio_info.audio_len > 0)SDL_Delay(0.5);
#endif}av_packet_unref(&packet);}else {break;}}av_frame_free(&pFrame);// 结束video_state->isEnd = 1;return 0;
}

第四步：视频解码（子线程）+播放（主线程）

说视频的解码和播放之前，先提一点：SDL的主窗口操作是需要在主线程中进行的。因此我们不能再解码子线程中直接渲染SDL窗口，否则会造成内存泄漏。知道这个知识之后，更能理解接下来的流程分析。

我们视频解码播放拆成两个部分：解码+播放

第一部分：解码子线程，在子线程中完成解码，通过标识符的方式通知到主线程帧已更新，并渲染出来。
第二部分：主线程播放，循环监听子线程的通知标识，并更新窗口帧进行显示。

代码如下：

视频解码子线程

/**视频线程*/
int video_thread(void *arg) {/**1. 从视频pkt队列中读出packet2. 送入解码器解码并取出3. 使用SDL进行渲染4. 根据pts计算延迟SDL_DELAY*/VideoState* 	video_state = (VideoState*) arg;PacketQueue* 	video_queue = video_state->vQueue;AVCodecContext* pCodecCtx = video_state->vCodecCtx;AVFrame* 		out_frame = video_state->videoParam->out_frame;AVPacket packet;AVFrame* pFrame = av_frame_alloc();AVRational time_base = video_state->formatCtx->streams[video_state->videoStream]->time_base;int64_t av_start_time = av_gettime();							// 开始播放时间(ms*1000)int64_t frame_delay = av_q2d(time_base) * AV_TIME_BASE;		// pts单位(ms*1000)int64_t frame_start_time = av_gettime();for (;;) {if (video_queue->size > 0) {packet_queue_pop(video_queue, &packet);// 将packet写入编解码器int ret = avcodec_send_packet(pCodecCtx, &packet);// 从解码器中取出原始帧while (!avcodec_receive_frame(pCodecCtx, pFrame)) {// 帧格式转化，转为YUV420Psws_scale(video_state->swsCtx,						// sws_context转换(uint8_t const * const *)pFrame->data,	// 输入 datapFrame->linesize,							// 输入 每行数据的大小（对齐）0,										// 输入 Y轴位置pCodecCtx->height,						// 输入 heightout_frame->data,							// 输出 dataout_frame->linesize);						// 输出 linesize// 帧更新video_state->videoParam->frame_update = 1;// 计算延迟int64_t pts = pFrame->pts;											// ptsint64_t actual_playback_time = av_start_time + pts * frame_delay;	// 实际播放时间int64_t current_time = av_gettime();if (actual_playback_time > current_time) {SDL_Delay((Uint32)(actual_playback_time-current_time)/1000);	// 延迟当前时间和实际播放时间}video_state->videoParam->index++;printf("第%i帧 | 属于%s | pts为%d | 时长为%.2fms | 实际播放点为%.2fs | 预期播放点为%.2fs\n ",video_state->videoParam->index,get_frame_type(pFrame),(int)pFrame->pts,(double)(av_gettime() - frame_start_time)/1000,(double)(av_gettime() - av_start_time)/AV_TIME_BASE,pFrame->pts * av_q2d(time_base));frame_start_time = av_gettime();}av_packet_unref(&packet);} else {break;}}av_frame_free(&pFrame);// 结束video_state->isEnd = 1;return 1;
}

渲染主线程

while (!video_state->isEnd) {// 处理事件（必须由主线程执行）while (SDL_PollEvent(&event)) {if (event.type == SDL_QUIT) {video_state->isEnd = 1;}}if (video_state->videoParam->frame_update) {// 将AVFrame的数据写入到texture中，然后渲染后windows上rect.x = 0;rect.y = 0;rect.w = video_state->vCodecCtx->width;rect.h = video_state->vCodecCtx->height;out_frame = video_state->videoParam->out_frame;// 更新纹理SDL_UpdateYUVTexture(texture, &rect,out_frame->data[0], out_frame->linesize[0],	// 	Yout_frame->data[1], out_frame->linesize[1],	// 	Uout_frame->data[2], out_frame->linesize[2]);	//  V// 渲染页面SDL_RenderClear(renderer);SDL_RenderCopy(renderer, texture, NULL, NULL);SDL_RenderPresent(renderer);// 重置标志video_state->videoParam->frame_update = 0;}}

完整代码

sample_player.h

//
//  sample_player.h
//  learning
//
//  Created by chenhuaiyi on 2025/2/26.
//#ifndef sample_player_h
#define sample_player_h#include <stdio.h>
// ffmpeg
#include "libavcodec/avcodec.h"
#include "libswresample/swresample.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavutil/imgutils.h"
#include "libavutil/time.h"
#include "libavutil/fifo.h"
#include "libavutil/channel_layout.h"
// SDL
#include "SDL.h"
#include "SDL_thread.h"/**宏定义*/
#define USE_SDL 1typedef struct MyAVPacketList {AVPacket 		*pkt;int 			serial;
} MyAVPacketList;/**packet队列*/
typedef struct PacketQueue {AVFifo* 		pkt_list;		// fifo队列int 			size;			// 队列大小SDL_mutex*		mutex;			// 互斥信号量SDL_cond*		cond;			// 条件变量，阻塞线程
} PacketQueue;/**数据类型定义*/
typedef struct AudioInfo{Uint32  		audio_len;		// 缓冲区长度Uint8*			audio_pos;		// 缓冲区起始地址指针
} AudioInfo;/**语音输出参数*/
typedef struct AudioParam {AVChannelLayout 	out_channel_layout;			// layoutint 				out_nb_samples;				// 每一帧的样本数enum AVSampleFormat out_sample_fmt;				// 格式int 				out_sample_rate;			// 采样率int 				out_channels;				// 输出通道数int					index;						// 音频帧总数int 				out_buffer_size;			// 音频输出缓冲区大小uint8_t* 			out_buffer;					// 音频输出缓冲区
} AudioParam;/**视频输出参数*/
typedef struct VideoParam {int 				width;						// 宽int 				height;						// 高enum AVPixelFormat	pix_fmt;					// 格式 YUV420Pint 				num_bytes;					// 单帧字节数int					index;						// 视频帧总数AVFrame*			out_frame;					// 输出帧int					frame_update;				// 帧更新标识
} VideoParam;/**全局参数*/
typedef struct VideoState {AVFormatContext* 	formatCtx;			// format上下文int				 	audioStream;		// 音频流索引AVCodecContext*  	aCodecCtx;			// 音频codec上下文const AVCodec*	 	aCodec;				// 音频解码器AudioParam*		 	audioParam;			// 音频参数SwrContext*			swrCtx;				// 音频上线文转换int					videoStream;		// 视频流索引AVCodecContext*  	vCodecCtx;			// 视频codec上新闻const AVCodec*	 	vCodec;				// 视频解码器VideoParam*			videoParam;			// 视频参数struct SwsContext* 	swsCtx;				// 视频上下文转换PacketQueue*		aQueue;				// 音频pkt队列PacketQueue*		vQueue;				// 视频pkt队列int					isEnd;				// 结束标志
} VideoState;/**全局变量*/
extern AudioInfo audio_info;#endif /* sample_player_h */

utils.h

//
//  utils.h
//  sample_player
//
//  Created by chenhuaiyi on 2025/2/27.
//#ifndef utils_h
#define utils_h#include "sample_player.h"int init_video_state(VideoState** video_state);int destory_video_state(VideoState** video_state);int packet_queue_push(PacketQueue* q, AVPacket* pkt);int packet_queue_init(PacketQueue** q, size_t max_size);int packet_queue_pop(PacketQueue* q, AVPacket* pkt);void packet_queue_destroy(PacketQueue** q);char* get_frame_type(AVFrame* frame);#endif /* utils_h */

manager.h

//
//  manager.h
//  sample_player
//
//  Created by chenhuaiyi on 2025/2/27.
//#ifndef manager_h
#define manager_h#include "sample_player.h"/**音频输出信息设置*/
int audio_output_set(VideoState* video_state);/**视频输出信息设置*/
int video_output_set(VideoState* video_state);/**音频SDL初始化*/
int audio_sdl_set(VideoState* video_state, SDL_AudioSpec* wanted_spec, void (*fn)(void*, Uint8*, int));/**视频SDL初始化*/
int video_sdl_set(VideoState* video_state, SDL_Window** window, SDL_Renderer** renderer, SDL_Texture** texture);#endif /* manager_h */

utils.c

//
//  utils.c
//  sample_player
//
//  Created by chenhuaiyi on 2025/2/27.
//#include "utils.h"/**初始化VideoState*/
int init_video_state(VideoState** video_state) {*video_state = av_malloc(sizeof(VideoState));(*video_state)->formatCtx = avformat_alloc_context();(*video_state)->audioStream = 0;(*video_state)->aCodecCtx = avcodec_alloc_context3(NULL);(*video_state)->audioParam = av_malloc(sizeof(AudioParam));(*video_state)->videoStream = 0;(*video_state)->vCodecCtx = avcodec_alloc_context3(NULL);(*video_state)->videoParam = av_malloc(sizeof(VideoParam));(*video_state)->videoParam->frame_update = 0;/** pkt队列初始化 */(*video_state)->aQueue = av_malloc(sizeof(PacketQueue));packet_queue_init(&(*video_state)->aQueue, 1);(*video_state)->vQueue = av_malloc(sizeof(PacketQueue));packet_queue_init(&(*video_state)->vQueue, 1);(*video_state)->isEnd = 0;return 1;
}/**销毁VideoState*/
int destory_video_state(VideoState** video_state){swr_free(&(*video_state)->swrCtx);avcodec_free_context(&(*video_state)->aCodecCtx);av_free((*video_state)->audioParam->out_buffer);av_free((*video_state)->audioParam);sws_freeContext((*video_state)->swsCtx);avcodec_free_context(&(*video_state)->vCodecCtx);av_frame_free(&(*video_state)->videoParam->out_frame);av_free((*video_state)->videoParam);/** 队列释放 */packet_queue_destroy(&(*video_state)->aQueue);packet_queue_destroy(&(*video_state)->vQueue);if ((*video_state)->formatCtx != NULL) {avformat_close_input(&(*video_state)->formatCtx);(*video_state)->formatCtx = NULL;}av_free(*video_state);return 1;
}/**初始化队列*/
int packet_queue_init(PacketQueue** q, size_t max_size) {// 创建一个 AVFifo 队列，每个元素的大小为 sizeof(AVPacket)*q = av_malloc(sizeof(PacketQueue));(*q)->pkt_list = av_fifo_alloc2(max_size, sizeof(MyAVPacketList), AV_FIFO_FLAG_AUTO_GROW);(*q)->size = 0;(*q)->mutex = SDL_CreateMutex();(*q)->cond = SDL_CreateCond();if (!(*q)->pkt_list) {return -1;}return 0;
}/**写入队列*/
int packet_queue_push(PacketQueue* q, AVPacket* pkt) {MyAVPacketList pNode;if (!q || !pkt) {return -1;}AVPacket* pkt1 = av_packet_alloc();if (!pkt1) {av_packet_unref(pkt);return -1;}SDL_LockMutex(q->mutex);av_packet_ref(pkt1, pkt);pNode.pkt = pkt1;// 将 pkt 压入队列if (av_fifo_write(q->pkt_list, &pNode, 1) < 0) {SDL_UnlockMutex(q->mutex);return -1;}q->size++;SDL_CondSignal(q->cond);SDL_UnlockMutex(q->mutex);return 0;
}/**弹出队列*/
int packet_queue_pop(PacketQueue* q, AVPacket* pkt) {if (!q || !pkt) {return -1;}SDL_LockMutex(q->mutex);MyAVPacketList pNode;// 从队列中弹出一个元素, 没找到则阻塞线程，等待生产者释放if (av_fifo_read(q->pkt_list, &pNode, 1) < 0) {SDL_CondWait(q->cond, q->mutex);}q->size--;av_packet_move_ref(pkt, pNode.pkt);av_packet_free(&pNode.pkt);SDL_UnlockMutex(q->mutex);return 0;
}/**销毁队列*/
void packet_queue_destroy(PacketQueue** q) {if ((*q) && (*q)->pkt_list) {// 释放队列中的所有 AVPacketMyAVPacketList pNode;SDL_LockMutex((*q)->mutex);while (av_fifo_read((*q)->pkt_list, &pNode, 1) >= 0) {av_packet_free(&pNode.pkt);  // 释放 AVPacket 的资源}SDL_UnlockMutex((*q)->mutex);// 释放 AVFifo 队列(*q)->size = 0;av_fifo_freep2(&(*q)->pkt_list);SDL_DestroyMutex((*q)->mutex);SDL_DestroyCond((*q)->cond);av_free(*q);}
}/**获取帧类型*/
char* get_frame_type(AVFrame* frame) {switch (frame->pict_type) {case AV_PICTURE_TYPE_I:return "I";break;case AV_PICTURE_TYPE_P:return "P";break;case AV_PICTURE_TYPE_B:return "B";break;case AV_PICTURE_TYPE_S:return "S";break;case AV_PICTURE_TYPE_SI:return "SI";break;case AV_PICTURE_TYPE_SP:return "SP";break;case AV_PICTURE_TYPE_BI:return "BI";break;default:return "N";break;}
}

manager.c

//
//  manager.c
//  sample_player
//
//  Created by chenhuaiyi on 2025/2/27.
//#include "manager.h"/**音频输出信息构建*/
int audio_output_set(VideoState* video_state) {AudioParam* audio_param = video_state->audioParam;// 输出用到的信息av_channel_layout_default(&audio_param->out_channel_layout, 2);audio_param->out_nb_samples = video_state->aCodecCtx->frame_size;			// 编解码器每个帧需要处理或者输出的采样点的大小 AAC:1024  MP3:1152audio_param->out_sample_fmt = AV_SAMPLE_FMT_S16;							// 采样格式audio_param->out_sample_rate = 44100;										// 采样率audio_param->out_channels = audio_param->out_channel_layout.nb_channels;	// 通道数// 获取需要使用的缓冲区大小 -> 通道数，单通道样本数，位深 1024(单帧处理的采样点)*2(双通道)*2(16bit对应2字节)audio_param->out_buffer_size = av_samples_get_buffer_size(NULL, audio_param->out_channels,audio_param->out_nb_samples,audio_param->out_sample_fmt, 1);// 分配缓冲区空间audio_param->out_buffer = NULL;av_samples_alloc(&audio_param->out_buffer, NULL, audio_param->out_channels,audio_param->out_nb_samples, audio_param->out_sample_fmt, 1);return 1;
}/**视频输出信息构建*/
int video_output_set(VideoState* video_state) {VideoParam* video_param = video_state->videoParam;// 基础信息video_param->width = video_state->vCodecCtx->width;video_param->height = video_state->vCodecCtx->height;video_param->pix_fmt = AV_PIX_FMT_YUV420P;// 计算单帧大小, 分配单帧内存video_param->num_bytes = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, video_param->width, video_param->height, 1);video_param->out_frame = av_frame_alloc();av_image_alloc(video_param->out_frame->data, video_param->out_frame->linesize,video_param->width, video_param->height, AV_PIX_FMT_YUV420P, 1);return 1;
}/**音频SDL初始化*/
int audio_sdl_set(VideoState* video_state, SDL_AudioSpec* wanted_spec, void (*fn)(void*, Uint8*, int)) {AudioParam* audio_param = video_state->audioParam;wanted_spec->freq = audio_param->out_sample_rate;					// 采样率wanted_spec->format = AUDIO_S16SYS;									// 采样格式 16bitwanted_spec->channels = audio_param->out_channels;					// 通道数wanted_spec->silence = 0;wanted_spec->samples = audio_param->out_nb_samples;					// 单帧处理的采样点wanted_spec->callback = fn;											// 回调函数wanted_spec->userdata = video_state->aCodecCtx;						// 回调函数的参数return 1;
}/**视频SDL初始化*/
int video_sdl_set(VideoState* video_state, SDL_Window** window, SDL_Renderer** renderer, SDL_Texture** texture){AVCodecContext* pCodecCtx = video_state->vCodecCtx;/** 窗口 */*window = SDL_CreateWindow("SDL2 window",SDL_WINDOWPOS_CENTERED,SDL_WINDOWPOS_CENTERED,pCodecCtx->width,pCodecCtx->height,SDL_WINDOW_SHOWN);if (!*window) {printf("SDL_CreateWindow Error: %s\n", SDL_GetError());SDL_Quit();return 1;}/** 渲染 */*renderer = SDL_CreateRenderer(*window,-1,SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);if (!*renderer) {printf("SDL_CreateRenderer Error: %s\n", SDL_GetError());SDL_DestroyWindow(*window);SDL_Quit();return 1;}/** 纹理 */*texture = SDL_CreateTexture(*renderer,SDL_PIXELFORMAT_YV12,SDL_TEXTUREACCESS_STREAMING,pCodecCtx->width,pCodecCtx->height);return 1;
}

main.c

//
//  main.c
//  sample_player
//
//  Created by chenhuaiyi on 2025/2/26.
//#include "utils.h"
#include "manager.h"AudioInfo audio_info;/* udata: 传入的参数* stream: SDL音频缓冲区* len: SDL音频缓冲区大小* 回调函数*/
void fill_audio(void *udata, Uint8 *stream, int len){SDL_memset(stream, 0, len);			// 必须重置，不然全是电音!!!if(audio_info.audio_len==0){					// 有音频数据时才调用return;}len = (len>audio_info.audio_len ? audio_info.audio_len : len);	// 最多填充缓冲区大小的数据SDL_MixAudio(stream, audio_info.audio_pos, len, SDL_MIX_MAXVOLUME);audio_info.audio_pos += len;audio_info.audio_len -= len;
}/**音频线程*/
int audio_thread(void *arg) {/**1. 从packet_queue队列中取出packet2. 将packet进行解码3. 写入到sdl的缓冲区中*/ 	VideoState* video_state = (VideoState*) arg;AudioParam* audio_param = video_state->audioParam;PacketQueue* queue = video_state->aQueue;audio_param->index = 0;AVRational time_base = video_state->formatCtx->streams[video_state->audioStream]->time_base;int64_t av_start_time = av_gettime();								// 播放开始时间戳AVPacket 	packet;int 		ret;AVFrame* 	pFrame = av_frame_alloc();for(;;) {if (queue->size > 0) {packet_queue_pop(queue, &packet);// 将packet写入编解码器ret = avcodec_send_packet(video_state->aCodecCtx, &packet);if ( ret < 0 ) {printf("send packet error\n");return -1;}// 获取解码后的帧while (!avcodec_receive_frame(video_state->aCodecCtx, pFrame)) {// 格式转化swr_convert(video_state->swrCtx, &audio_param->out_buffer, audio_param->out_buffer_size,(const uint8_t **)pFrame->data, pFrame->nb_samples);audio_param->index++;printf("第%d帧 | pts:%lld | 帧大小(采样点):%d | 实际播放点%.2fs | 预期播放点%.2fs\n",audio_param->index,packet.pts,packet.size,(double)(av_gettime() - av_start_time)/AV_TIME_BASE,pFrame->pts * av_q2d(time_base));#if USE_SDL// 设置读取的音频数据audio_info.audio_len = audio_param->out_buffer_size;audio_info.audio_pos = (Uint8 *) audio_param->out_buffer;// 等待SDL播放完成while(audio_info.audio_len > 0)SDL_Delay(0.5);
#endif}av_packet_unref(&packet);}else {break;}}av_frame_free(&pFrame);// 结束video_state->isEnd = 1;return 0;
}/**视频线程*/
int video_thread(void *arg) {/**1. 从视频pkt队列中读出packet2. 送入解码器解码并取出3. 使用SDL进行渲染4. 根据pts计算延迟SDL_DELAY*/VideoState* 	video_state = (VideoState*) arg;PacketQueue* 	video_queue = video_state->vQueue;AVCodecContext* pCodecCtx = video_state->vCodecCtx;AVFrame* 		out_frame = video_state->videoParam->out_frame;AVPacket packet;AVFrame* pFrame = av_frame_alloc();AVRational time_base = video_state->formatCtx->streams[video_state->videoStream]->time_base;int64_t av_start_time = av_gettime();							// 开始播放时间(ms*1000)int64_t frame_delay = av_q2d(time_base) * AV_TIME_BASE;		// pts单位(ms*1000)int64_t frame_start_time = av_gettime();for (;;) {if (video_queue->size > 0) {packet_queue_pop(video_queue, &packet);// 将packet写入编解码器int ret = avcodec_send_packet(pCodecCtx, &packet);if (ret < 0) {printf("packet resolve error!");break;}// 从解码器中取出原始帧while (!avcodec_receive_frame(pCodecCtx, pFrame)) {// 帧格式转化，转为YUV420Psws_scale(video_state->swsCtx,						// sws_context转换(uint8_t const * const *)pFrame->data,	// 输入 datapFrame->linesize,							// 输入 每行数据的大小（对齐）0,										// 输入 Y轴位置pCodecCtx->height,						// 输入 heightout_frame->data,							// 输出 dataout_frame->linesize);						// 输出 linesize// 帧更新video_state->videoParam->frame_update = 1;// 计算延迟int64_t pts = pFrame->pts;											// ptsint64_t actual_playback_time = av_start_time + pts * frame_delay;	// 实际播放时间int64_t current_time = av_gettime();if (actual_playback_time > current_time) {SDL_Delay((Uint32)(actual_playback_time-current_time)/1000);	// 延迟当前时间和实际播放时间}video_state->videoParam->index++;printf("第%i帧 | 属于%s | pts为%d | 时长为%.2fms | 实际播放点为%.2fs | 预期播放点为%.2fs\n ",video_state->videoParam->index,get_frame_type(pFrame),(int)pFrame->pts,(double)(av_gettime() - frame_start_time)/1000,(double)(av_gettime() - av_start_time)/AV_TIME_BASE,pFrame->pts * av_q2d(time_base));frame_start_time = av_gettime();}av_packet_unref(&packet);} else {break;}}av_frame_free(&pFrame);// 结束video_state->isEnd = 1;return 1;
}int main(int argc, char* argv[])
{VideoState* 		video_state;AudioParam*			audio_param;VideoParam*			video_param;SDL_Event 			event;SDL_Rect      		rect;if(argc < 2) {fprintf(stderr, "Usage: test <file>\n");exit(1);}/** 初始化函数 */init_video_state(&video_state);audio_param = video_state->audioParam;video_param = video_state->videoParam;avformat_network_init();// 1. 打开视频文件，获取格式上下文if(avformat_open_input(&video_state->formatCtx, argv[1], NULL, NULL)!=0){printf("Couldn't open input stream.\n");return -1;}// 2. 对文件探测流信息if(avformat_find_stream_info(video_state->formatCtx, NULL) < 0){printf("Couldn't find stream information.\n");return -1;}// 打印信息av_dump_format(video_state->formatCtx, 0, argv[1], 0);// 3. 找到对应的 音频流/视频流 索引video_state->audioStream=-1;video_state->videoStream=-1;for(int i=0; i < video_state->formatCtx->nb_streams; i++) {if(video_state->formatCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_AUDIO){video_state->audioStream=i;}if (video_state->formatCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_VIDEO) {video_state->videoStream=i;}}if(video_state->audioStream==-1){printf("Didn't find a audio stream.\n");return -1;}if (video_state->videoStream==-1) {printf("Didn't find a video stream.\n");return -1;}// 4. 将 音频流/视频流 编码参数写入上下文AVCodecParameters* aCodecParam = video_state->formatCtx->streams[video_state->audioStream]->codecpar;avcodec_parameters_to_context(video_state->aCodecCtx, aCodecParam);
//	avcodec_parameters_free(&aCodecParam); 这个是不需要手动释放的AVCodecParameters* vCodecParam = video_state->formatCtx->streams[video_state->videoStream]->codecpar;avcodec_parameters_to_context(video_state->vCodecCtx, vCodecParam);
//	avcodec_parameters_free(&vCodecParam);// 5. 查找流的编码器video_state->aCodec = avcodec_find_decoder(video_state->aCodecCtx->codec_id);if(video_state->aCodec==NULL){printf("Audio codec not found.\n");return -1;}video_state->vCodec = avcodec_find_decoder(video_state->vCodecCtx->codec_id);if(video_state->vCodec==NULL){printf("Video codec not found.\n");return -1;}// 6. 打开流的编解码器if(avcodec_open2(video_state->aCodecCtx, video_state->aCodec, NULL)<0){printf("Could not open audio codec.\n");return -1;}if(avcodec_open2(video_state->vCodecCtx, video_state->vCodec, NULL)<0){printf("Could not open video codec.\n");return -1;}/** 音频输出信息构建 */audio_output_set(video_state);/** 视频输出信息构建 */video_output_set(video_state);//	SDL 初始化
#if USE_SDLif(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {printf( "Could not initialize SDL - %s\n", SDL_GetError());return -1;}// 在 main 函数开始处添加SDL_SetHint(SDL_HINT_VIDEO_MAC_FULLSCREEN_SPACES, "0");SDL_SetHint(SDL_HINT_MAC_BACKGROUND_APP, "1");/** 初始化音频SDL设备 */SDL_AudioSpec wanted_spec;//	audio_sdl_set(video_state, &wanted_spec, fill_audio);wanted_spec.freq = audio_param->out_sample_rate;					// 采样率wanted_spec.format = AUDIO_S16SYS;									// 采样格式 16bitwanted_spec.channels = audio_param->out_channels;					// 通道数wanted_spec.silence = 0;wanted_spec.samples = audio_param->out_nb_samples;					// 单帧处理的采样点wanted_spec.callback = fill_audio;									// 回调函数wanted_spec.userdata = video_state->aCodecCtx;						// 回调函数的参数/** 初始化视频SDL设备 */SDL_Window*       window = NULL;SDL_Renderer*     renderer = NULL;SDL_Texture*      texture= NULL;//	video_sdl_set(video_state, &window, &renderer, &texture);/** 窗口 */window = SDL_CreateWindow("SDL2 window",SDL_WINDOWPOS_CENTERED,SDL_WINDOWPOS_CENTERED,video_state->vCodecCtx->width,video_state->vCodecCtx->height,SDL_WINDOW_SHOWN);if (!window) {printf("SDL_CreateWindow Error: %s\n", SDL_GetError());SDL_Quit();return 1;}/** 渲染 */renderer = SDL_CreateRenderer(window,-1,SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);if (!renderer) {printf("SDL_CreateRenderer Error: %s\n", SDL_GetError());SDL_DestroyWindow(window);SDL_Quit();return 1;}/** 纹理 */texture = SDL_CreateTexture(renderer,SDL_PIXELFORMAT_YV12,SDL_TEXTUREACCESS_STREAMING,video_state->vCodecCtx->width,video_state->vCodecCtx->height);// 打开音频播放器if (SDL_OpenAudio(&wanted_spec, NULL)<0) {printf("can't open audio.\n");return -1;}#endif// 音频上下文格式转换swr_alloc_set_opts2(&video_state->swrCtx,&audio_param->out_channel_layout,			// 输出layoutaudio_param->out_sample_fmt,				// 输出格式audio_param->out_sample_rate,				// 输出采样率&video_state->aCodecCtx->ch_layout,			// 输入layoutvideo_state->aCodecCtx->sample_fmt,			// 输入格式video_state->aCodecCtx->sample_rate,		// 输入采样率0, NULL);swr_init(video_state->swrCtx);// 视频上下文格式转换video_state->swsCtx = sws_getContext(video_state->vCodecCtx->width,			// src 宽video_state->vCodecCtx->height,		// src 高video_state->vCodecCtx->pix_fmt,		// src 格式video_param->width,					// dst 宽video_param->height,					// dst 高video_param->pix_fmt,					// dst 格式SWS_BILINEAR,NULL,NULL,NULL);// 开始播放SDL_PauseAudio(0);int64_t av_start_time = av_gettime();	// 播放开始时间戳AVPacket* packet = av_packet_alloc();	// packet初始化// 循环1: 从文件中读取packetwhile(av_read_frame(video_state->formatCtx, packet)>=0){/** 写入音频pkt队列 */if(packet->stream_index==video_state->audioStream){packet_queue_push(video_state->aQueue, packet);}/** 写入视频pkt队列 */if (packet->stream_index==video_state->videoStream) {packet_queue_push(video_state->vQueue, packet);}av_packet_unref(packet);SDL_PollEvent(&event);switch(event.type) {case SDL_QUIT:SDL_Quit();exit(0);break;default:break;}}printf("audio queue.size=%d\n", video_state->aQueue->size);// 创建一个线程并启动SDL_CreateThread(audio_thread, "audio_thread", video_state);SDL_CreateThread(video_thread, "video_thread", video_state);//	video_thread(video_state);AVFrame* out_frame = NULL;while (!video_state->isEnd) {// 处理事件（必须由主线程执行）while (SDL_PollEvent(&event)) {if (event.type == SDL_QUIT) {video_state->isEnd = 1;}}if (video_state->videoParam->frame_update) {// 将AVFrame的数据写入到texture中，然后渲染后windows上rect.x = 0;rect.y = 0;rect.w = video_state->vCodecCtx->width;rect.h = video_state->vCodecCtx->height;out_frame = video_state->videoParam->out_frame;// 更新纹理SDL_UpdateYUVTexture(texture, &rect,out_frame->data[0], out_frame->linesize[0],	// 	Yout_frame->data[1], out_frame->linesize[1],	// 	Uout_frame->data[2], out_frame->linesize[2]);	//  V// 渲染页面SDL_RenderClear(renderer);SDL_RenderCopy(renderer, texture, NULL, NULL);SDL_RenderPresent(renderer);// 重置标志video_state->videoParam->frame_update = 0;}}// 打印参数printf("格式: %s\n", video_state->formatCtx->iformat->name);printf("时长: %lld us\n", video_state->formatCtx->duration);printf("音频持续时长为 %.2f，音频帧总数为 %d\n", (double)(av_gettime()-av_start_time)/AV_TIME_BASE, audio_param->index);printf("码率: %lld\n", video_state->formatCtx->bit_rate);printf("编码器: %s (%s)\n", video_state->aCodecCtx->codec->long_name, avcodec_get_name(video_state->aCodecCtx->codec_id));printf("通道数: %d\n", video_state->aCodecCtx->ch_layout.nb_channels);printf("采样率: %d \n", video_state->aCodecCtx->sample_rate);printf("单通道每帧的采样点数目: %d\n", video_state->aCodecCtx->frame_size);printf("pts单位(ms*1000): %.2f\n", av_q2d(video_state->formatCtx->streams[video_state->audioStream]->time_base) * AV_TIME_BASE);// 释放空间av_packet_free(&packet);#if USE_SDLSDL_CloseAudio();SDL_DestroyTexture(texture);SDL_DestroyRenderer(renderer);SDL_DestroyWindow(window);SDL_Quit();
#endifdestory_video_state(&video_state);return 0;
}

结语和展望

终于做完了，恭喜你，完成了一个非常粗糙，而且有很多问题的简单音视频播放器。接下来几期，我们跟着大家一起对这个简单的播放器进行优化。当然我也是个小萌新，所以一步一步来嘛哈哈。先抛出几个问题：

1. 时钟同步怎么做
2. 如何边读出packet，边解码frame并播放
3. 我们如何对输出的解码帧进行转化

ps. 鼓励大家阅读ffplay源码，所有的问题都能迎刃而解，哈哈哈！