安卓Camera2用ImageReader获取NV21源码分析

以前如何得到Camera预览流回调

• 可以通过如下方法，得到一路预览回调流 Camera#setPreviewCallbackWithBuffer(Camera.PreviewCallback)，
• 可以通过如下方法，设置回调数据的格式，比如 ImageFormat.NV21 Camera.Parameters#setPreviewForma
• 从API21开始，android.hardware.Camera 相关API 已经被谷歌废弃，
  当我们切换到Camera2相关后，如何得到一路NV21回调流呢？

如何使用Camera2得到预览流的回调

一般我们使用的方法如下，

private final ImageReader.OnImageAvailableListener mOnImageAvailableListener = reader -> {Image image = reader.acquireNextImage();// do sth with readerimage.close();
};mImageReader = ImageReader.newInstance(mPreviewSize.getWidth(), mPreviewSize.getHeight(),ImageFormat.YUV_420_888, 1);
mImageReader.setOnImageAvailableListener(mOnImageAvailableListener, mCameraHandler);
// 看起来是不是跟Camera2拍照很相似，关键在配流时需要添加如下代码
mPreviewRequestBuilder.addTarget(mImageReader.getSurface());

这样我们就可以通过 Image 读取回调的内容，内容是啥呢？

The order of planes in the array returned by Image#getPlanes() is guaranteed such that plane #0 is always Y, plane #1 is always U (Cb), and plane #2 is always V (Cr).

Image#getPlanes 返回三个数组，plane 0 是Y，plane 1 是U (Cb)，plane 2 是 V (Cr)
我们可以把他们转给NV21格式，如何转换呢？
可以参考如下答案
stackoverflow camera2-conversion-from-yuv-420-888-to-nv21

是否可以通过ImageReader直接得到NV21回调呢

上面我们设置的是YUV_420_888，是否可以直接改成NV21呢?
不行，在 ImageReader 我们可以看到如下，会直接闪退

if (format == ImageFormat.NV21) { throw new IllegalArgumentException( "NV21 format is not supported")
}

ImageReader数据流回调的流程源码分析

相关数据流源码分析如下

private static void postEventFromNative(Object selfRef) {....if (executor != null && listener != null && isReaderValid) {executor.execute(new Runnable() {@Overridepublic void run() {listener.onImageAvailable(ir);}});}
}// frameworks/base/media/jni/android_media_ImageReader.cpp 中部分源码如下
static struct {jfieldID mNativeContext;jmethodID postEventFromNative;
} gImageReaderClassInfo;
gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
{ALOGV("%s: frame available", __FUNCTION__);bool needsDetach = false;JNIEnv* env = getJNIEnv(&needsDetach);if (env != NULL) {env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);} else {ALOGW("onFrameAvailable event will not posted");}if (needsDetach) {detachJNI();}
}// frameworks/native/libs/gui/ConsumerBase.cpp
void ConsumerBase::onFrameAvailable(const BufferItem& item) {CB_LOGV("onFrameAvailable");sp<FrameAvailableListener> listener;{ // scope for the lockMutex::Autolock lock(mFrameAvailableMutex);listener = mFrameAvailableListener.promote();}if (listener != nullptr) {CB_LOGV("actually calling onFrameAvailable");listener->onFrameAvailable(item);}
}// APP 通过 ImageReader 设置 OnImageAvailableListener 回调拿到 android.media.ImageReader
ImageReader.acquireNextImage() 拿到 android.media.Image
public Image acquireNextImage() {// Initialize with reader format, but can be overwritten by native if the image// format is different from the reader format.SurfaceImage si = new SurfaceImage(mFormat);int status = acquireNextSurfaceImage(si);return si;
}
private int acquireNextSurfaceImage(SurfaceImage si) {....status = nativeImageSetup(si);....
}// frameworks.base\media\jni\android_media_ImageReader.cpp
{"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup }static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image){JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();BufferItem* buffer = ctx->getBufferItem();status_t res = bufferConsumer->acquireBuffer(buffer, 0);ctx->returnBufferItem(buffer);
}List<BufferItem*> mBuffers;
void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {buffer->mGraphicBuffer = nullptr;mBuffers.push_back(buffer);
}
frameworks/native/libs/gui/include/gui/BufferItem.handroid.media.ImageReader$SurfaceImage
@Override
public Plane[] getPlanes() {throwISEIfImageIsInvalid();if (mPlanes == null) {// mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat);mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat);}// Shallow copy is fine.return mPlanes.clone();
}
{"nativeCreatePlanes",      "(IIJ)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",  (void*)Image_createSurfacePlanes },static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,int numPlanes, int readerFormat, uint64_t ndkReaderUsage){jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, /*initial_element*/NULL);LockedImage lockedImg = LockedImage();    Image_getLockedImage(env, thiz, &lockedImg, ndkReaderUsage); // 把前面的 BufferItem 转为 LockedImagefor (int i = 0; i < numPlanes; i++) {if (!Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,&pData, &dataSize, &pixelStride, &rowStride)) {return NULL;}byteBuffer = env->NewDirectByteBuffer(pData, dataSize);// Finally, create this SurfacePlane.jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);}}
static void ImageReader_classInit(JNIEnv* env, jclass clazz) {gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>","(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
}static bool Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,pixelStride, rowStride);if (res != OK) {jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException","Pixel format: 0x%x is unsupported", buffer->flexFormat);return false;}return true;
}// frameworks/base/media/jni/android_media_Utils.cpp
status_t lockImageFromBuffer(BufferItem* bufferItem, uint32_t inUsage,int fenceFd, LockedImage* outputImage) {...status_t res = lockImageFromBuffer(bufferItem->mGraphicBuffer, inUsage, bufferItem->mCrop,fenceFd, outputImage);...
}
status_t lockImageFromBuffer(sp<GraphicBuffer> buffer, uint32_t inUsage,const Rect& rect, int fenceFd, LockedImage* outputImage) {ALOGV("%s: Try to lock the GraphicBuffer", __FUNCTION__);if (buffer == nullptr || outputImage == nullptr) {ALOGE("Input BufferItem or output LockedImage is NULL!");return BAD_VALUE;}if (isFormatOpaque(buffer->getPixelFormat())) {ALOGE("Opaque format buffer is not lockable!");return BAD_VALUE;}void* pData = NULL;android_ycbcr ycbcr = android_ycbcr();status_t res;int format = buffer->getPixelFormat();int flexFormat = format;if (isPossiblyYUV(format)) {res = buffer->lockAsyncYCbCr(inUsage, rect, &ycbcr, fenceFd);if (res != OK) {ALOGW("lockAsyncYCbCr failed with error %d (format = 0x%x)", res, format);}pData = ycbcr.y;flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;}// lockAsyncYCbCr for YUV is unsuccessful.if (pData == NULL) {res = buffer->lockAsync(inUsage, rect, &pData, fenceFd);if (res != OK) {ALOGE("Lock buffer failed!");return res;}if (isPossibly10BitYUV(format)&& OK == extractP010Gralloc4PlaneLayout(buffer, pData, format, outputImage)) {ALOGV("%s: Successfully locked the P010 image", __FUNCTION__);return OK;}}outputImage->data = reinterpret_cast<uint8_t*>(pData);outputImage->width = buffer->getWidth();outputImage->height = buffer->getHeight();outputImage->format = format;outputImage->flexFormat = flexFormat;outputImage->stride =(ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();outputImage->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb);outputImage->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr);outputImage->chromaStride = static_cast<uint32_t>(ycbcr.cstride);outputImage->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step);ALOGV("%s: Successfully locked the image from the GraphicBuffer", __FUNCTION__);// Crop, transform, scalingMode, timestamp, and frameNumber should be set by caller,// and cann't be set them here.return OK;
}
status_t getLockedImageInfo(LockedImage* buffer, int idx,int32_t containerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {ALOGV("%s", __FUNCTION__);LOG_ALWAYS_FATAL_IF(buffer == NULL, "Input buffer is NULL!!!");LOG_ALWAYS_FATAL_IF(base == NULL, "base is NULL!!!");LOG_ALWAYS_FATAL_IF(size == NULL, "size is NULL!!!");LOG_ALWAYS_FATAL_IF(pixelStride == NULL, "pixelStride is NULL!!!");LOG_ALWAYS_FATAL_IF(rowStride == NULL, "rowStride is NULL!!!");LOG_ALWAYS_FATAL_IF((idx >= IMAGE_MAX_NUM_PLANES) || (idx < 0), "idx (%d) is illegal", idx);ALOGV("%s: buffer: %p", __FUNCTION__, buffer);uint32_t dataSize, ySize, cSize, cStride;uint32_t pStride = 0, rStride = 0;uint8_t *cb, *cr;uint8_t *pData = NULL;int bytesPerPixel = 0;dataSize = ySize = cSize = cStride = 0;int32_t fmt = buffer->flexFormat;bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, containerFormat);fmt = applyFormatOverrides(fmt, containerFormat);switch (fmt) {case HAL_PIXEL_FORMAT_YCbCr_420_888:// Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.if (buffer->width % 2 != 0) {ALOGE("YCbCr_420_888: width (%d) should be a multiple of 2", buffer->width);return BAD_VALUE;}if (buffer->height % 2 != 0) {ALOGE("YCbCr_420_888: height (%d) should be a multiple of 2", buffer->height);return BAD_VALUE;}if (buffer->width <= 0) {ALOGE("YCbCr_420_888: width (%d) should be a > 0", buffer->width);return BAD_VALUE;}if (buffer->height <= 0) {ALOGE("YCbCr_420_888: height (%d) should be a > 0", buffer->height);return BAD_VALUE;}pData =(idx == 0) ?buffer->data :(idx == 1) ?buffer->dataCb :buffer->dataCr;// only map until last pixelif (idx == 0) {pStride = 1;rStride = buffer->stride;dataSize = buffer->stride * (buffer->height - 1) + buffer->width;} else {pStride = buffer->chromaStep;rStride = buffer->chromaStride;dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +buffer->chromaStep * (buffer->width / 2 - 1) + 1;}break;// NV21case HAL_PIXEL_FORMAT_YCrCb_420_SP:// Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.if (buffer->width % 2 != 0) {ALOGE("YCrCb_420_SP: width (%d) should be a multiple of 2", buffer->width);return BAD_VALUE;}if (buffer->height % 2 != 0) {ALOGE("YCrCb_420_SP: height (%d) should be a multiple of 2", buffer->height);return BAD_VALUE;}if (buffer->width <= 0) {ALOGE("YCrCb_420_SP: width (%d) should be a > 0", buffer->width);return BAD_VALUE;}if (buffer->height <= 0) {ALOGE("YCrCb_420_SP: height (%d) should be a > 0", buffer->height);return BAD_VALUE;}cr = buffer->data + (buffer->stride * buffer->height);cb = cr + 1;// only map until last pixelySize = buffer->width * (buffer->height - 1) + buffer->width;cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;pData =(idx == 0) ?buffer->data :(idx == 1) ?cb:cr;dataSize = (idx == 0) ? ySize : cSize;pStride = (idx == 0) ? 1 : 2;rStride = buffer->width;break;case HAL_PIXEL_FORMAT_YV12:// Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.if (buffer->width % 2 != 0) {ALOGE("YV12: width (%d) should be a multiple of 2", buffer->width);return BAD_VALUE;}if (buffer->height % 2 != 0) {ALOGE("YV12: height (%d) should be a multiple of 2", buffer->height);return BAD_VALUE;}if (buffer->width <= 0) {ALOGE("YV12: width (%d) should be a > 0", buffer->width);return BAD_VALUE;}if (buffer->height <= 0) {ALOGE("YV12: height (%d) should be a > 0", buffer->height);return BAD_VALUE;}// Y and C stride need to be 16 pixel aligned.LOG_ALWAYS_FATAL_IF(buffer->stride % 16,"Stride is not 16 pixel aligned %d", buffer->stride);ySize = buffer->stride * buffer->height;cStride = ALIGN(buffer->stride / 2, 16);cr = buffer->data + ySize;cSize = cStride * buffer->height / 2;cb = cr + cSize;pData =(idx == 0) ?buffer->data :(idx == 1) ?cb :cr;dataSize = (idx == 0) ? ySize : cSize;pStride = 1;rStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);break;case HAL_PIXEL_FORMAT_YCBCR_P010:if (buffer->height % 2 != 0) {ALOGE("YCBCR_P010: height (%d) should be a multiple of 2", buffer->height);return BAD_VALUE;}if (buffer->width <= 0) {ALOGE("YCBCR_P010: width (%d) should be a > 0", buffer->width);return BAD_VALUE;}if (buffer->height <= 0) {ALOGE("YCBCR_P010: height (%d) should be a > 0", buffer->height);return BAD_VALUE;}if (buffer->dataCb && buffer->dataCr) {pData =(idx == 0) ?buffer->data :(idx == 1) ?buffer->dataCb :buffer->dataCr;// only map until last pixelif (idx == 0) {pStride = 2;rStride = buffer->stride;dataSize = buffer->stride * (buffer->height - 1) + buffer->width * 2;} else {pStride = buffer->chromaStep;rStride = buffer->chromaStride;dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +buffer->chromaStep * (buffer->width / 2);}break;}ySize = (buffer->stride * 2) * buffer->height;cSize = ySize / 2;pStride = (idx == 0) ? 2 : 4;cb = buffer->data + ySize;cr = cb + 2;pData = (idx == 0) ?  buffer->data : (idx == 1) ?  cb : cr;dataSize = (idx == 0) ? ySize : cSize;rStride = buffer->stride * 2;break;case HAL_PIXEL_FORMAT_Y8:// Single plane, 8bpp.LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);pData = buffer->data;dataSize = buffer->stride * buffer->height;pStride = 1;rStride = buffer->stride;break;case HAL_PIXEL_FORMAT_Y16:bytesPerPixel = 2;// Single plane, 16bpp, strides are specified in pixels, not in bytesLOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);pData = buffer->data;dataSize = buffer->stride * buffer->height * bytesPerPixel;pStride = bytesPerPixel;rStride = buffer->stride * 2;break;case HAL_PIXEL_FORMAT_BLOB:// Used for JPEG data, height must be 1, width == size, single plane.LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);// When RGBA override is being used, buffer height will be equal to widthif (usingRGBAOverride) {LOG_ALWAYS_FATAL_IF(buffer->height != buffer->width,"RGBA override BLOB format buffer should have height == width");} else {LOG_ALWAYS_FATAL_IF(buffer->height != 1,"BLOB format buffer should have height value 1");}pData = buffer->data;dataSize = Image_getBlobSize(buffer, usingRGBAOverride);pStride = 0;rStride = 0;break;case HAL_PIXEL_FORMAT_RAW16:// Single plane 16bpp bayer data.bytesPerPixel = 2;LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);pData = buffer->data;dataSize = buffer->stride * buffer->height * bytesPerPixel;pStride = bytesPerPixel;rStride = buffer->stride * 2;break;case HAL_PIXEL_FORMAT_RAW_OPAQUE:// Used for RAW_OPAQUE data, height must be 1, width == size, single plane.LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);LOG_ALWAYS_FATAL_IF(buffer->height != 1,"RAW_PRIVATE should has height value one but got %d", buffer->height);pData = buffer->data;dataSize = buffer->width;pStride = 0; // RAW OPAQUE doesn't have pixel striderStride = 0; // RAW OPAQUE doesn't have row stridebreak;case HAL_PIXEL_FORMAT_RAW10:// Single plane 10bpp bayer data.LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);LOG_ALWAYS_FATAL_IF(buffer->width % 4,"Width is not multiple of 4 %d", buffer->width);LOG_ALWAYS_FATAL_IF(buffer->height % 2,"Height is not even %d", buffer->height);LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8),"stride (%d) should be at least %d",buffer->stride, buffer->width * 10 / 8);pData = buffer->data;dataSize = buffer->stride * buffer->height;pStride = 0;rStride = buffer->stride;break;case HAL_PIXEL_FORMAT_RAW12:// Single plane 10bpp bayer data.LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);LOG_ALWAYS_FATAL_IF(buffer->width % 4,"Width is not multiple of 4 %d", buffer->width);LOG_ALWAYS_FATAL_IF(buffer->height % 2,"Height is not even %d", buffer->height);LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 12 / 8),"stride (%d) should be at least %d",buffer->stride, buffer->width * 12 / 8);pData = buffer->data;dataSize = buffer->stride * buffer->height;pStride = 0;rStride = buffer->stride;break;case HAL_PIXEL_FORMAT_RGBA_8888:case HAL_PIXEL_FORMAT_RGBX_8888:// Single plane, 32bpp.bytesPerPixel = 4;LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);pData = buffer->data;dataSize = buffer->stride * buffer->height * bytesPerPixel;pStride = bytesPerPixel;rStride = buffer->stride * 4;break;case HAL_PIXEL_FORMAT_RGB_565:// Single plane, 16bpp.bytesPerPixel = 2;LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);pData = buffer->data;dataSize = buffer->stride * buffer->height * bytesPerPixel;pStride = bytesPerPixel;rStride = buffer->stride * 2;break;case HAL_PIXEL_FORMAT_RGB_888:// Single plane, 24bpp.bytesPerPixel = 3;LOG_ALWAYS_FATAL_IF(idx != 0, "Wrong index: %d", idx);pData = buffer->data;dataSize = buffer->stride * buffer->height * bytesPerPixel;pStride = bytesPerPixel;rStride = buffer->stride * 3;break;default:ALOGV("%s: unrecognized format 0x%x", __FUNCTION__, fmt);return BAD_VALUE;}*base = pData;*size = dataSize;*pixelStride = pStride;*rowStride = rStride;return OK;
}

以上部分注释待添加，待完善整理

---