ffmpeg7.0 flv支持hdr

ffmpeg7.0 flv支持hdr

自从ffmpeg6.0应用enhance rtmp支持h265/av1的flv格式后，7.0迎来了flv的hdr能力。本文介绍ffmpeg7.0如何支持hdr in flv。

如果对enhance rtmp如何支持h265不了解，推荐详解Enhanced-RTMP支持H.265

1. enhance rtmp关于hdr

文档enhance-rtmp-v2.md中，metadata-frame章节规定相关支持HDR部分。

定义一个新的VideoPacketType.Metadata，其内容为AMF格式的metadata(AMF是一种简便的描述格式，非常节省内存)，具体内部有HDR相关的colorInfo数据(每次hdr参数更新的时候，就发送该metadata)。

type ColorInfo = {colorConfig: {// number of bits used to record the color channels for each pixelbitDepth:                 number, // SHOULD be 8, 10 or 12//// colorPrimaries, transferCharacteristics and matrixCoefficients are defined // in ISO/IEC 23091-4/ITU-T H.273. The values are an index into // respective tables which are described in “Colour primaries”, // "Transfer characteristics" and "Matrix coefficients" sections. // It is RECOMMENDED to provide these values.//// indicates the chromaticity coordinates of the source color primariescolorPrimaries:           number, // enumeration [0-255]// opto-electronic transfer characteristic function (ex. PQ, HLG)transferCharacteristics:  number, // enumeration [0-255]// matrix coefficients used in deriving luma and chroma signalsmatrixCoefficients:       number, // enumeration [0-255]},hdrCll: {//// maximum value of the frame average light level// (in 1 cd/m2) of the entire playback sequence//maxFall:  number,     // [0.0001-10000]//// maximum light level of any single pixel (in 1 cd/m2)// of the entire playback sequence//maxCLL:   number,     // [0.0001-10000]},//// The hdrMdcv object defines mastering display (i.e., where// creative work is done during the mastering process) color volume (a.k.a., mdcv)// metadata which describes primaries, white point and min/max luminance. The// hdrMdcv object SHOULD be provided.//// Specification of the metadata along with its ranges adhere to the// ST 2086:2018 - SMPTE Standard (except for minLuminance see// comments below)//hdrMdcv: {//// Mastering display color volume (mdcv) xy Chromaticity Coordinates within CIE// 1931 color space.//// Values SHALL be specified with four decimal places. The x coordinate SHALL// be in the range [0.0001, 0.7400]. The y coordinate SHALL be // in the range [0.0001, 0.8400].//redX:         number,redY:         number,greenX:       number,greenY:       number,blueX:        number,blueY:        number,whitePointX:  number,whitePointY:  number,//// max/min display luminance of the mastering display (in 1 cd/m2 ie. nits)//// note: ST 2086:2018 - SMPTE Standard specifies minimum display mastering// luminance in multiples of 0.0001 cd/m2.// // For consistency we specify all values// in 1 cd/m2. Given that a hypothetical perfect screen has a peak brightness// of 10,000 nits and a black level of .0005 nits we do not need to// switch units to 0.0001 cd/m2 to increase resolution on the lower end of the// minLuminance property. The ranges (in nits) mentioned below suffice// the theoretical limit for Mastering Reference Displays and adhere to the// SMPTE ST 2084 standard (a.k.a., PQ) which is capable of representing full gamut// of luminance level.//maxLuminance: number,     // [5-10000]minLuminance: number,     // [0.0001-5]},
}

2. flv mux

在libavformat/flvenc.c中，新增flv_write_metadata_packet函数，用于添加metadata。
flv的数据结构:

typedef struct FLVContext {.....int metadata_pkt_written;//0: 还未写过，需要写；1: 写过了，不需要写。
} FLVContext;

在函数flv_write_metadata_packet中实现，见注释：

static void flv_write_metadata_packet(AVFormatContext *s, AVCodecParameters *par, unsigned int ts) {FLVContext *flv = s->priv_data;//不需要写入，就返回；可以通过设置这个标志变量来使能/去使能更新写metadataif (flv->metadata_pkt_written)return;//支持h265, av1, vp9if (par->codec_id == AV_CODEC_ID_HEVC || par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {......//写入tag头，标识其为视频avio_w8(pb, FLV_TAG_TYPE_VIDEO); //write video tag typemetadata_size_pos = avio_tell(pb);avio_wb24(pb, 0 + flags_size);put_timestamp(pb, ts); //ts = pkt->dts, genavio_wb24(pb, flv->reserved);//根据enhance rtmp标志，写入FLV_IS_EX_HEADER标识，和fourCC的字段(hvc1 or av01 or vp09)if (par->codec_id == AV_CODEC_ID_HEVC) {avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD); // ExVideoTagHeader mode with PacketTypeMetadataavio_write(pb, "hvc1", 4);} else if (par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD);avio_write(pb, par->codec_id == AV_CODEC_ID_AV1 ? "av01" : "vp09", 4);}//下面为写入AMF格式的hdr相关的colorInfo数据avio_w8(pb, AMF_DATA_TYPE_STRING);put_amf_string(pb, "colorInfo");avio_w8(pb, AMF_DATA_TYPE_OBJECT);put_amf_string(pb, "colorConfig");  // colorConfigavio_w8(pb, AMF_DATA_TYPE_OBJECT);if (par->color_trc != AVCOL_TRC_UNSPECIFIED &&par->color_trc < AVCOL_TRC_NB) {put_amf_string(pb, "transferCharacteristics");  // color_trcput_amf_double(pb, par->color_trc);}if (par->color_space != AVCOL_SPC_UNSPECIFIED &&par->color_space < AVCOL_SPC_NB) {put_amf_string(pb, "matrixCoefficients"); // colorspaceput_amf_double(pb, par->color_space);}if (par->color_primaries != AVCOL_PRI_UNSPECIFIED &&par->color_primaries < AVCOL_PRI_NB) {put_amf_string(pb, "colorPrimaries"); // color_primariesput_amf_double(pb, par->color_primaries);}put_amf_string(pb, "");avio_w8(pb, AMF_END_OF_OBJECT);if (lightMetadata) {put_amf_string(pb, "hdrCll");avio_w8(pb, AMF_DATA_TYPE_OBJECT);put_amf_string(pb, "maxFall");put_amf_double(pb, lightMetadata->MaxFALL);put_amf_string(pb, "maxCLL");put_amf_double(pb, lightMetadata->MaxCLL);put_amf_string(pb, "");avio_w8(pb, AMF_END_OF_OBJECT);}if (displayMetadata && (displayMetadata->has_primaries || displayMetadata->has_luminance)) {put_amf_string(pb, "hdrMdcv");avio_w8(pb, AMF_DATA_TYPE_OBJECT);if (displayMetadata->has_primaries) {put_amf_string(pb, "redX");put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][0]));put_amf_string(pb, "redY");put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][1]));put_amf_string(pb, "greenX");put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][0]));put_amf_string(pb, "greenY");put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][1]));put_amf_string(pb, "blueX");put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][0]));put_amf_string(pb, "blueY");put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][1]));put_amf_string(pb, "whitePointX");put_amf_double(pb, av_q2d(displayMetadata->white_point[0]));put_amf_string(pb, "whitePointY");put_amf_double(pb, av_q2d(displayMetadata->white_point[1]));}if (displayMetadata->has_luminance) {put_amf_string(pb, "maxLuminance");put_amf_double(pb, av_q2d(displayMetadata->max_luminance));put_amf_string(pb, "minLuminance");put_amf_double(pb, av_q2d(displayMetadata->min_luminance));}put_amf_string(pb, "");avio_w8(pb, AMF_END_OF_OBJECT);}put_amf_string(pb, "");avio_w8(pb, AMF_END_OF_OBJECT);flv->metadata_pkt_written = 1;//标识写过了}
}

其中HDR的数据来源为AVCodecParameters *par数据结构中的内容：

typedef struct AVCodecParameters {..../*** Video only. Additional colorspace characteristics.*/enum AVColorRange                  color_range;enum AVColorPrimaries              color_primaries;enum AVColorTransferCharacteristic color_trc;enum AVColorSpace                  color_space;enum AVChromaLocation              chroma_location;/** 类型: AV_PKT_DATA_CONTENT_LIGHT_LEVEL, 数据: AVContentLightMetadata* lightMetadata* 类型: AV_PKT_DATA_MASTERING_DISPLAY_METADATA, 数据: AVMasteringDisplayMetadata *displayMetadata*/AVPacketSideData *coded_side_data;....
}

3. flv demux

解析函数在libavformat/flvdec.c文件中，函数amf_parse_object中。

static int amf_parse_object(AVFormatContext *s, AVStream *astream,AVStream *vstream, const char *key,int64_t max_pos, int depth)
{FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;......if (meta_video_color) {if (amf_type == AMF_DATA_TYPE_NUMBER ||amf_type == AMF_DATA_TYPE_BOOL) {if (!strcmp(key, "colorPrimaries")) {meta_video_color->primaries = num_val;} else if (!strcmp(key, "transferCharacteristics")) {meta_video_color->transfer_characteristics = num_val;} else if (!strcmp(key, "matrixCoefficients")) {meta_video_color->matrix_coefficients = num_val;} else if (!strcmp(key, "maxFall")) {meta_video_color->max_fall = num_val;} else if (!strcmp(key, "maxCLL")) {meta_video_color->max_cll = num_val;} else if (!strcmp(key, "redX")) {meta_video_color->mastering_meta.r_x = num_val;} else if (!strcmp(key, "redY")) {meta_video_color->mastering_meta.r_y = num_val;} else if (!strcmp(key, "greenX")) {meta_video_color->mastering_meta.g_x = num_val;} else if (!strcmp(key, "greenY")) {meta_video_color->mastering_meta.g_y = num_val;} else if (!strcmp(key, "blueX")) {meta_video_color->mastering_meta.b_x = num_val;} else if (!strcmp(key, "blueY")) {meta_video_color->mastering_meta.b_y = num_val;} else if (!strcmp(key, "whitePointX")) {meta_video_color->mastering_meta.white_x = num_val;} else if (!strcmp(key, "whitePointY")) {meta_video_color->mastering_meta.white_y = num_val;} else if (!strcmp(key, "maxLuminance")) {meta_video_color->mastering_meta.max_luminance = num_val;} else if (!strcmp(key, "minLuminance")) {meta_video_color->mastering_meta.min_luminance = num_val;}}}......
}

4. 感谢Zhu Pengfei的提交

Author: Zhu Pengfei <[e-mail]>
Date:   Mon Mar 4 21:52:04 2024 +0800avformat/flvenc: support enhanced flv PacketTypeMetadataSigned-off-by: Zhu Pengfei <[e-mail]>Signed-off-by: Steven Liu <[e-mail]>
``