FFmpeg与OpenCV联合开发

本文讲述如何利用FFmpeg SDK与OpenCV 从RTSP流中获取图像（OpenCV MAT 对象格式）。

一，构造RTSP视频流

因为是在本机实验，所以我自己构造了一个RTSP流。如果你有现成的RTSP流也可以的。
实验用的源视频是黑神话·悟空的《云宫讯音》。

自己产生一个RTSP视频流得需要一个RTSP流媒体服务器。目前可以用的RTSP流媒体服务器有两个，分别是ZLMediaKit和MediaMTX两个。ZLMediaKit需要编译，我编译好了。MediaMTX下载即用。这里我选了MediaMTX，下载下来就可以用，比较轻量化。

1，启动流媒体服务器

在MediaMTX的安装目录下启动即可。

2，启动FFmpeg推流

ffmpeg -re -stream_loop -1 -i “wukong.mp4” -vcodec copy -acodec copy -f rtsp -rtsp_transport tcp rtsp://192.168.76.189:8554/live/test

这是流的相关信息：
Input #0, rtsp, from ‘rtsp://192.168.76.189:8554/live/test’:f=0/0
Metadata:
title : No Name
Duration: N/A, start: 0.000000, bitrate: N/A
Stream #0:0: Video: h264 (Main), yuv420p(tv, bt709, progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn
Stream #0:1: Audio: aac (LC), 44100 Hz, stereo, fltp

3，用FFplay播放一下

二，源代码

#include <stdio.h>
#include <thread>
#include <string>
//#include <io.h>
//#include <vector>//#define __STDC_CONSTANT_MACROS
#include <iostream>
using namespace std;
extern "C"
{
#include "libavformat/avformat.h"
#include "libavcodec/avcodec.h"
#include "libavutil/time.h"
//#include "libavutil/log.h"
//#include "libavutil/mathematics.h"
#include "libswscale/swscale.h"
//#include "libavutil/imgutils.h"
}#include <opencv2/opencv.hpp>using namespace cv;int main(int argc, char* argv[])
{Mat pCvMat;//const string sourceWindow = "test";//namedWindow(sourceWindow,1);//cout<<avcodec_version()<<endl;//cout<<FFMPEG_VERSION<<endl;unsigned codecVer = avcodec_version();int ver_major, ver_minor, ver_micro;ver_major = (codecVer >> 16) & 0xff;ver_minor = (codecVer >> 8) & 0xff;ver_micro = (codecVer) & 0xff;printf("Current ffmpeg version is:    ,avcodec version is: %d=%d.%d.%d\n",codecVer, ver_major, ver_minor, ver_micro);int startTime = 0;                           // 记录播放开始int currentFrame = 0;                           // 当前帧序号double fps = 0;                                 // 帧率double interval = 0;                            // 帧间隔const AVInputFormat* p_ifmt_v = NULL;// ffmpeg相关变量预先定义与分配AVFormatContext* pAVFormatContext = 0;          // ffmpeg的全局上下文，所有ffmpeg操作都需要AVStream* pAVStream = 0;                        // ffmpeg流信息AVCodecContext* pAVCodecContext = avcodec_alloc_context3(NULL);            // ffmpeg编码上下文const AVCodec* pAVCodec = 0;                          // ffmpeg编码器AVPacket* pAVPacket = 0;                        // ffmpag单帧数据包AVFrame* pAVFrame = 0;                          // ffmpeg单帧缓存//AVFrame *pAVFrameBGR24 = 0;                     // ffmpeg单帧缓存转换颜色空间后的缓存struct SwsContext* pSwsContext = 0;             // ffmpeg编码数据格式转换AVDictionary* pAVDictionary = 0;                // ffmpeg数据字典，用于配置一些编码器属性等//unsigned char * outBuffer = 0;                           // 解码后的数据存放缓存区//const char* url_v = "video.sdp";int ret = 0;                                    // 函数执行结果int videoIndex = -1;                            // 音频流所在的序号//int numBytes = 0;                               // 解码后的数据长度pAVFormatContext = avformat_alloc_context();    // 分配//pAVFormatContext->flags |= AVFMT_NOFILE;//添加白名单，这里很重要，如果不申请内存，在avformat_close_input中会宕//pAVFormatContext->protocol_whitelist = (char*)av_malloc(sizeof("file,udp,rtp"));//memcpy(pAVFormatContext->protocol_whitelist, "file,udp,rtp", sizeof("file,udp,rtp"));//pAVPacket = av_packet_alloc();                  // 分配pAVPacket = (AVPacket*)av_malloc(sizeof(AVPacket));pAVFrame = av_frame_alloc();                    // 分配//pAVFrameBGR24 = av_frame_alloc();               // 分配p_ifmt_v = av_find_input_format("rtsp");//if(!pAVFormatContext || !pAVPacket || !pAVFrame || !pAVFrameBGR24)if (!pAVFormatContext || !pAVPacket || !pAVFrame){cout << "Failed to alloc" << endl;system("pause");return -1;}// 步骤一：注册所有容器和编解码器（也可以只注册一类，如注册容器、注册编码器等）//av_register_all();avformat_network_init();// 步骤二：打开文件(ffmpeg成功则返回0)//std::string src = "rtsp://192.168.76.215/live/test";std::string src = argv[1];cout << "打开:" << src << endl;AVDictionary* format_opts = NULL;av_dict_set(&format_opts, "rtsp_transport", "tcp", 0); //设置推流的方式，默认udp。ret = avformat_open_input(&pAVFormatContext, src.c_str(), p_ifmt_v, &format_opts);//nullptr//ret = avformat_open_input(&pAVFormatContext, src.c_str(), p_ifmt_v, nullptr);if (ret){cout << "Failed" << endl;system("pause");return -1;}// 步骤三：探测流媒体信息ret = avformat_find_stream_info(pAVFormatContext, 0);if (ret < 0){cout << "Failed to avformat_find_stream_info(pAVFormatContext, 0)" << endl;system("pause");return -1;}printf("******************\n");av_dump_format(pAVFormatContext, 0, src.c_str(), 0);printf("******************\n");// 步骤四：提取流信息,提取视频信息for (int index = 0; index < pAVFormatContext->nb_streams; index++){if (pAVFormatContext->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO){cout << "流序号:" << index << "\n类型为:" << "AVMEDIA_TYPE_VIDEO" << endl;//pAVCodecContext = pAVFormatContext->streams[index]->codecpar->;avcodec_parameters_to_context(pAVCodecContext, pAVFormatContext->streams[index]->codecpar);pAVStream = pAVFormatContext->streams[index];videoIndex = index;break;}}if (videoIndex == -1 || !pAVCodecContext){cout << "Failed to find video stream" << endl;system("pause");return -1;}// 步骤五：对找到的视频流寻解码器pAVCodec = avcodec_find_decoder(pAVCodecContext->codec_id);if (!pAVCodec){cout << "Fialed to avcodec_find_decoder(pAVCodecContext->codec_id):"<< pAVCodecContext->codec_id << endl;system("pause");return -1;}// 步骤六：打开解码器// 设置缓存大小 1024000byteav_dict_set(&pAVDictionary, "buffer_size", "8192000", 0);// 设置超时时间 20sav_dict_set(&pAVDictionary, "stimeout", "20000000", 0);// 设置最大延时 3sav_dict_set(&pAVDictionary, "max_delay", "30000000", 0);// 设置打开方式 tcp/udpav_dict_set(&pAVDictionary, "rtsp_transport", "tcp", 0);//ret = avcodec_open2(pAVCodecContext, pAVCodec, &pAVDictionary);ret = avcodec_open2(pAVCodecContext, pAVCodec, &pAVDictionary);if (ret){cout << "Failed to avcodec_open2(pAVCodecContext, pAVCodec, pAVDictionary)" << endl;system("pause");return -1;}// 显示视频相关的参数信息（编码上下文）cout << "比特率:" << pAVCodecContext->bit_rate << endl;cout << "宽高:" << pAVCodecContext->width << "x" << pAVCodecContext->height << endl;cout << "格式:" << pAVCodecContext->pix_fmt << endl;  // AV_PIX_FMT_YUV420P 0cout << "帧率分母:" << pAVCodecContext->time_base.den << endl;cout << "帧率分子:" << pAVCodecContext->time_base.num << endl;cout << "帧率分母:" << pAVStream->avg_frame_rate.den << endl;cout << "帧率分子:" << pAVStream->avg_frame_rate.num << endl;cout << "总时长:" << pAVStream->duration / 10000.0 << "s" << endl;cout << "总帧数:" << pAVStream->nb_frames << endl;// 有总时长的时候计算帧率（较为准确）//    fps = pAVStream->nb_frames / (pAVStream->duration / 10000.0);//    interval = pAVStream->duration / 10.0 / pAVStream->nb_frames;// 没有总时长的时候，使用分子和分母计算fps = pAVStream->avg_frame_rate.num * 1.0f / pAVStream->avg_frame_rate.den;interval = 1 * 1000 / fps;cout << "平均帧率:" << fps << endl;cout << "帧间隔:" << interval << "ms" << endl;// 步骤七：对拿到的原始数据格式进行缩放转换为指定的格式高宽大小pAVCodecContext->pix_fmt = (AVPixelFormat)AV_PIX_FMT_YUV420P;pSwsContext = sws_getContext(pAVCodecContext->width,pAVCodecContext->height,pAVCodecContext->pix_fmt,pAVCodecContext->width,pAVCodecContext->height,AV_PIX_FMT_BGR24,SWS_FAST_BILINEAR,0,0,0);cout << "sws_getContext!" << endl;pCvMat = Mat(pAVCodecContext->height, pAVCodecContext->width, CV_8UC3);int cvLinesizes[1];cvLinesizes[0] = pCvMat.step1();// 步骤八：读取一帧数据的数据包//av_free_packet(pAVPacket);//av_packet_unref(pAVPacket);int frame_count = 0;//cout << "now av_read_frame!" << endl;//namedWindow("img", 0);while (av_read_frame(pAVFormatContext, pAVPacket) >= 0){if (pAVPacket->stream_index == videoIndex){//cout << "av_read_frame!" << endl;// 步骤八：对读取的数据包进行解码ret = avcodec_send_packet(pAVCodecContext, pAVPacket);//ret = avcodec_decode_video2(pAVCodecContext, pAVFrame, &got_picture, pAVPacket);if (ret){cout << "Failed to avcodec_send_packet(pAVCodecContext, pAVPacket) ,ret ="<< ret << endl;break;}//cout << "avcodec_send_packet!" << endl;av_frame_unref(pAVFrame);while (!avcodec_receive_frame(pAVCodecContext, pAVFrame)){//pCvMat.data=NULL;sws_scale(pSwsContext,(const uint8_t* const*)pAVFrame->data,pAVFrame->linesize,0,pAVCodecContext->height,&pCvMat.data,cvLinesizes);//cout << "avcodec_receive_frame!" << endl;frame_count++;if (frame_count % int(25) == 0) {//sprintf()//char * temp;//itoa(frame_count,temp,10);//string sss=temp;imwrite("./image/test_" + std::to_string(frame_count) + ".bmp", pCvMat);//imshow("img", pCvMat);cout << "SAVE: " << std::to_string(frame_count) << endl;}//threshold(pCvMat,pCvMat,128,255,THRESH_BINARY);//imshow(sourceWindow,pCvMat);//waitKey(1);av_frame_unref(pAVFrame);}// 下一帧currentFrame++;while (av_gettime() - startTime < currentFrame * interval){//Sleep(1);std::this_thread::sleep_for(std::chrono::milliseconds(1));}//cout << "current:" << currentFrame <<"," << time << av_gettime()- startTime <<endl;}//av_free_packet(pAVPacket);av_packet_unref(pAVPacket);}cout << "释放回收资源" << endl;if (pSwsContext){sws_freeContext(pSwsContext);pSwsContext = 0;cout << "sws_freeContext(pSwsContext)" << endl;}if (pAVFrame){av_frame_free(&pAVFrame);pAVFrame = 0;cout << "av_frame_free(pAVFrame)" << endl;}if (pAVPacket){//av_free_packet(pAVPacket);pAVPacket = 0;cout << "av_free_packet(pAVPacket)" << endl;}if (pAVCodecContext){avcodec_free_context(&pAVCodecContext);avcodec_close(pAVCodecContext);pAVCodecContext = 0;cout << "avcodec_close(pAVCodecContext);" << endl;}if (pAVFormatContext){avformat_close_input(&pAVFormatContext);avformat_free_context(pAVFormatContext);pAVFormatContext = 0;cout << "avformat_free_context(pAVFormatContext)" << endl;}pCvMat = NULL;system("pause");return 0;
}

三，运行一下

TestFFmpegOpencv_RTSP.exe rtsp://192.168.76.189:8554/live/test

四，结果

提取出的帧结果保存在image文件夹下：