java获取近期视频流关键帧与截图

1、背景

最近在做视频转发的开发时，遇到一个问题，前端订阅播放h264视频流时，有时会出现一段时间黑屏，经过测试发现是没有收到关键帧，只有第一帧是关键帧才能保证后续播放正常。所以后端需要实现一个功能，就是前端在进入播放页面时，后端把最近的一个关键帧发过去。

2、思路（环形缓存区）

后端接收到的视频流是一个个的字节数组，所以在接收时没法直接判断一帧的开始和结束，需要将最近的一段视频流截取出来，然后利用ffmpeg工具进行整体的解析和关键帧提取。
查看ffmpeg工具的代码，可以发现ffmpeg工具入参是inputstream，该工具会不断调用inputstream的read方法进行字节的读取。所以就想通过一个环形缓存区不断的记录最新的一段视频流数据。该环形缓存区再实现inputstream的接口，重写read方法，read读取的开始位置即是环形缓存区头，到环形缓存区的尾时自动结束。

3、依赖包

<!--   javacv    -->
<dependency><groupId>org.bytedeco</groupId><artifactId>javacpp</artifactId><version>1.4.3</version>
</dependency>
<dependency><groupId>org.bytedeco</groupId><artifactId>javacv</artifactId><version>1.4.3</version>
</dependency>
<dependency><groupId>org.bytedeco.javacpp-presets</groupId><artifactId>ffmpeg-platform</artifactId><version>4.0.2-1.4.3</version>
</dependency>

4、环形缓存区定义

public class CycleBufferInputStream extends InputStream {/************************************************  环形缓冲区  ***********************************************/private ByteBuffer buffer = null;private int readPos = 0; //将要读取的位置private int writePos = 0; //将要写入的位置private boolean isCycle = false; //判断是否已经形成一个环public CycleBufferInputStream(int capacity) {this.buffer = ByteBuffer.allocateDirect(capacity);}/*** 将字节数组以覆盖的方式放入环形缓冲区*/public void put(byte[] bytes) {int used = buffer.capacity() - buffer.position();if (used < bytes.length) {buffer.put(bytes, 0, used);buffer.clear();buffer.put(bytes, used, bytes.length - used);isCycle = true;} else if (used == bytes.length) {buffer.put(bytes, 0, used);buffer.clear();isCycle = true;} else {buffer.put(bytes, 0, bytes.length);}writePos = buffer.position();}/*** 定位读取的初始位置（执行inputstream 读取前，必须要先调用该方法）*/public void readPrepare() {if (buffer.capacity() == writePos || !isCycle) {readPos = 0;} else {readPos = buffer.position() + 1;}}/*************************************************** 输入流传输  ***************************************************//*** Reads the next byte of data from the input stream. The value byte is returned as an int in the range 0 to 255. If no byte is available because the end of the stream has been reached, the value -1 is returned. This method blocks until input data is available, the end of the stream is detected, or an exception is thrown.* A subclass must provide an implementation of this method.* Returns: the next byte of data, or -1 if the end of the stream is reached.* Throws: IOException – if an I/O error occurs.*/@Overridepublic int read() throws IOException {if (readPos == buffer.capacity()) readPos = 0;if (readPos == writePos) return -1;int value = buffer.get(readPos++);if (value < 0) value = value + 256;return value;}}

5、从环形缓存区提取关键帧

    /*** 从环形缓存环获取最近一帧关键帧字节数组* 这里返回的堆外内存，所以注意要及时进行内存释放* @param inputStream*/public static ByteBuffer dealVideo(InputStream inputStream)  {try {int j = 0;FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);ff.start();Frame frame = null;Frame last = null;while ((frame = ff.grabKeyFrame()) != null && frame.image != null) {last = frame.clone();System.out.println("获取一帧" + j++);}ff.stop();if (last != null) {System.out.println("获取最近的一个关键帧");ByteBuffer byteBuffer = (ByteBuffer)last.image[0];return byteBuffer;}} catch (Exception e) {log.error("提取最近一个关键帧异常\n",e);}return null;}

6、使用

    public static void main(String[] args) throws Exception {CycleBufferInputStream stream = new CycleBufferInputStream(1024 * 1024 * 10);FileInputStream fis = new FileInputStream("D:\\tmp-data\\1694511149969.h264");byte[] bytes = new byte[fis.available()];fis.read(bytes);stream.put(bytes);stream.readPrepare();dealVideo(stream);}

7、扩展：获取近期视频的截图

其实就是从近期的关键帧中提取出图片，关键代码如下：

    public static void dealImage(InputStream inputStream) {try {Java2DFrameConverter converter = new Java2DFrameConverter();FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);ff.start();int j = 0;Frame frame = null;Frame last = null;while ((frame = ff.grabImage()) != null) {last = frame.clone();System.out.println("获取一张图片" + j++);}ff.stop();if (last != null && last.image != null) {System.out.println("存储最后一张图片 ");BufferedImage fecthedImage = converter.getBufferedImage(last);File screenshotFile = new File("D:\\tmp-data\\", System.currentTimeMillis() + ".jpg");ImageIO.write(fecthedImage, "jpg", screenshotFile);}} catch (Exception e) {log.error("提取最近一个图片异常\n",e);}}

8、整体代码

package com.qq.utils;import lombok.extern.slf4j.Slf4j;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameConverter;import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.ByteBuffer;@Slf4j
public class CycleBufferInputStream extends InputStream {/************************************************  环形缓冲区  ***********************************************/private ByteBuffer buffer = null;private int readPos = 0; //将要读取的位置private int writePos = 0; //将要写入的位置private boolean isCycle = false; //判断是否已经形成一个环public CycleBufferInputStream(int capacity) {this.buffer = ByteBuffer.allocateDirect(capacity);}/*** 将字节数组以覆盖的方式放入环形缓冲区*/public void put(byte[] bytes) {int used = buffer.capacity() - buffer.position();if (used < bytes.length) {buffer.put(bytes, 0, used);buffer.clear();buffer.put(bytes, used, bytes.length - used);isCycle = true;} else if (used == bytes.length) {buffer.put(bytes, 0, used);buffer.clear();isCycle = true;} else {buffer.put(bytes, 0, bytes.length);}writePos = buffer.position();}/*** 定位读取的初始位置（执行inputstream方法前，必须要先调用该方法）*/public void readPrepare() {if (buffer.capacity() == writePos || !isCycle) {readPos = 0;} else {readPos = buffer.position() + 1;}}/*************************************************** 输入流传输  ***************************************************/@Overridepublic int read() throws IOException {if (readPos == buffer.capacity()) readPos = 0;if (readPos == writePos) return -1;int value = buffer.get(readPos++);if (value < 0) value = value + 256;return value;}/*** 从环形缓存环获取最近一帧关键帧字节数组* 这里返回的堆外内存，所以注意要及时进行内存释放* @param inputStream*/public static ByteBuffer dealVideo(InputStream inputStream)  {try {int j = 0;FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);ff.start();Frame frame = null;Frame last = null;while ((frame = ff.grabKeyFrame()) != null && frame.image != null) {last = frame.clone();System.out.println("获取一帧" + j++);}ff.stop();if (last != null) {System.out.println("获取最近的一个关键帧");ByteBuffer byteBuffer = (ByteBuffer)last.image[0];return byteBuffer;}} catch (Exception e) {log.error("提取最近一个关键帧异常\n",e);}return null;}public static void dealImage(InputStream inputStream) {try {Java2DFrameConverter converter = new Java2DFrameConverter();FFmpegFrameGrabber ff = new FFmpegFrameGrabber(inputStream);ff.start();int j = 0;Frame frame = null;Frame last = null;while ((frame = ff.grabImage()) != null) {last = frame.clone();System.out.println("获取一张图片" + j++);}ff.stop();if (last != null && last.image != null) {System.out.println("存储最后一张图片 ");BufferedImage fecthedImage = converter.getBufferedImage(last);File screenshotFile = new File("D:\\tmp-data\\", System.currentTimeMillis() + ".jpg");ImageIO.write(fecthedImage, "jpg", screenshotFile);}} catch (Exception e) {log.error("提取最近一个图片异常\n",e);}}}