深度学习之用CelebA_Spoof数据集搭建一个活体检测-训练好的模型用MNN来推理

一、模型转换准备

首先确保已完成PyTorch到ONNX的转换：深度学习之用CelebA_Spoof数据集搭建活体检测系统：模型验证与测试。这里有将PyTorch到ONNX格式的模型转换。

二、ONNX转MNN

使用MNN转换工具进行格式转换：具体的编译过程可以参考MNN的官方代码。MNN是一个轻量级的深度神经网络引擎，支持深度学习的推理与训练。适用于服务器/个人电脑/手机/嵌入式各类设备。

./MNNConvert -f ONNX --modelFile live_spoof.onnx --MNNModel live_spoof.mnn

三、C++推理工程搭建

工程结构

mnn_inference/
├── CMakeLists.txt
├── include/
│   ├── InferenceInit.h
│   └── LiveSpoofDetector.h
├── src/
│   ├── InferenceInit.cpp
│   ├── LiveSpoofDetector.cpp
│   └── CMakeLists.txt
└── third_party/MNN/

根目录下的 CMakeLists.txt

cmake_minimum_required(VERSION 3.12)
project(MNNInference)# 设置C++标准
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)# 查找OpenCV
find_package(OpenCV REQUIRED)# 包含第三方库MNN
set(MNN_DIR ${CMAKE_SOURCE_DIR}/third_party/MNN)
include_directories(${MNN_DIR}/include)# 添加子目录
add_subdirectory(src)# 主可执行文件
add_executable(mnn_inference_main src/main.cpp
)# 链接库
target_link_libraries(mnn_inference_mainPRIVATEinference_lib${MNN_DIR}/lib/libMNN.so${OpenCV_LIBS}
)# 安装规则
install(TARGETS mnn_inference_mainRUNTIME DESTINATION bin
)

src目录下的 CMakeLists.txt

# 添加库
add_library(inference_lib STATICInferenceInit.cppLiveSpoofDetector.cpp
)# 包含目录
target_include_directories(inference_libPUBLIC${CMAKE_SOURCE_DIR}/include${MNN_DIR}/include${OpenCV_INCLUDE_DIRS}
)# 编译选项
target_compile_options(inference_libPRIVATE-Wall-O3
)

核心实现代码

将MNN读取导入模型和一些mnn_session进行预处理的公共部分抽取出来，以后可以更换不同的模型，只需要给出特定的预处理。

// InferenceInit.h
#ifndef MNN_CORE_MNN_HANDLER_H
#define MNN_CORE_MNN_HANDLER_H
#include "MNN/Interpreter.hpp"
#include "MNN/MNNDefine.h"
#include "MNN/Tensor.hpp"
#include "MNN/ImageProcess.hpp"#include <iostream>
#include "opencv2/opencv.hpp"
#endif
#include "mylog.h"
#define LITEMNN_DEBUG
namespace mnncore
{class  BasicMNNHandler{protected:std::shared_ptr<MNN::Interpreter> mnn_interpreter;MNN::Session *mnn_session = nullptr;MNN::Tensor *input_tensor = nullptr; // assume single input.MNN::ScheduleConfig schedule_config;std::shared_ptr<MNN::CV::ImageProcess> pretreat; // init at subclassconst char *log_id = nullptr;const char *mnn_path = nullptr;const char *mnn_model_data = nullptr;//int mnn_model_size = 0;protected:const int num_threads; // initialize at runtime.int input_batch;int input_channel;int input_height;int input_width;int dimension_type;int num_outputs = 1;protected:explicit BasicMNNHandler(const std::string &_mnn_path, int _num_threads = 1);int initialize_handler();std::string turnHeadDataToString(std::string headData);virtual ~BasicMNNHandler();// un-copyableprotected:BasicMNNHandler(const BasicMNNHandler &) = delete; //BasicMNNHandler(BasicMNNHandler &&) = delete; //BasicMNNHandler &operator=(const BasicMNNHandler &) = delete; //BasicMNNHandler &operator=(BasicMNNHandler &&) = delete; //protected:virtual void transform(const cv::Mat &mat) = 0; // ? needed ?private:void print_debug_string();};
}

// InferenceInit.cpp
#include "mnn/core/InferenceInit.h"
namespace mnncore
{
BasicMNNHandler::BasicMNNHandler(const std::string &_mnn_path, int _num_threads) :log_id(_mnn_path.data()), mnn_path(_mnn_path.data()),num_threads(_num_threads)
{//initialize_handler();
}int BasicMNNHandler::initialize_handler()
{std::cout<<"load  Model from file: " << mnn_path << "\n";mnn_interpreter = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(mnn_path));myLog(ERROR_, "mnn_interpreter createFromFile done!");if (nullptr == mnn_interpreter) {std::cout << "load centerface failed." << std::endl;return -1;}// 2. init schedule_configschedule_config.numThread = (int) num_threads;MNN::BackendConfig backend_config;backend_config.precision = MNN::BackendConfig::Precision_Low; // default Precision_Highbackend_config.memory = MNN::BackendConfig::Memory_Low;backend_config.power = MNN::BackendConfig::Power_Low;schedule_config.backendConfig = &backend_config;// 3. create sessionmyLog(ERROR_, "createSession...");mnn_session = mnn_interpreter->createSession(schedule_config);// 4. init input tensormyLog(ERROR_, "getSessionInput...");input_tensor = mnn_interpreter->getSessionInput(mnn_session, nullptr);// 5. init input dimsinput_batch = input_tensor->batch();input_channel = input_tensor->channel();input_height = input_tensor->height();input_width = input_tensor->width();dimension_type = input_tensor->getDimensionType();myLog(ERROR_, "input_batch: %d, input_channel: %d, input_height: %d, input_width: %d, dimension_type: %d", input_batch, input_channel, input_height, input_width, dimension_type);// 6. resize tensor & session needed ???if (dimension_type == MNN::Tensor::CAFFE){// NCHWmnn_interpreter->resizeTensor(input_tensor, {input_batch, input_channel, input_height, input_width});mnn_interpreter->resizeSession(mnn_session);} // NHWCelse if (dimension_type == MNN::Tensor::TENSORFLOW){mnn_interpreter->resizeTensor(input_tensor, {input_batch, input_height, input_width, input_channel});mnn_interpreter->resizeSession(mnn_session);} // NC4HW4else if (dimension_type == MNN::Tensor::CAFFE_C4){
#ifdef LITEMNN_DEBUGstd::cout << "Dimension Type is CAFFE_C4, skip resizeTensor & resizeSession!\n";
#endif}// output countnum_outputs = (int)mnn_interpreter->getSessionOutputAll(mnn_session).size();
#ifdef LITEMNN_DEBUGthis->print_debug_string();
#endifreturn 0;
}BasicMNNHandler::~BasicMNNHandler()
{mnn_interpreter->releaseModel();if (mnn_session)mnn_interpreter->releaseSession(mnn_session);
}
void BasicMNNHandler::print_debug_string()
{std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n";std::cout << "=============== Input-Dims ==============\n";if (input_tensor) input_tensor->printShape();if (dimension_type == MNN::Tensor::CAFFE)std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n";else if (dimension_type == MNN::Tensor::TENSORFLOW)std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n";else if (dimension_type == MNN::Tensor::CAFFE_C4)std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n";std::cout << "=============== Output-Dims ==============\n";auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session);std::cout << "getSessionOutputAll done!\n";for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it){std::cout << "Output: " << it->first << ": ";it->second->printShape();}std::cout << "========================================\n";
}
} // namespace mnncore

主要的推理处理代码：
头文件声明

//LiveSpoofDetector.h
#include "mnn/core/InferenceInit.h"
#include <iostream>
#include <fstream>
#include <sstream>
#include<iterator>
#include <algorithm>#define RESIZE_LIVE_SPOOF_SIZE 112
using namespace mnncore;
namespace mnncv {class SqueezeNetLiveSpoof : public BasicMNNHandler{public:explicit SqueezeNetLiveSpoof(const std::string &model_path, int numThread = 1);~SqueezeNetLiveSpoof() override = default;// 保留原有函数int Init(const char* model_path);float detect_handler(const unsigned char* pData, int width, int height, int nchannel,  int mod);cv::Mat m_image;private:void initialize_pretreat();void transform(const cv::Mat &mat) override;std::vector<cv::Point2f> coord5points;const float meanVals_[3] = { 103.94f, 116.78f, 123.68f};const float normVals_[3] = {0.017f, 0.017f, 0.017f};};
}

函数定义：

// LiveSpoofDetector.cpp
#include "include/mnn/cv/RGB/LiveSpoofDetector.h"
#include <opencv2/opencv.hpp>using namespace mnncv;SqueezeNetLiveSpoof::SqueezeNetLiveSpoof(const std::string &model_path, int numThread) : BasicMNNHandler(model_path, numThread) {initialize_pretreat();
}int SqueezeNetLiveSpoof::Init(const char* model_path) {std::string model_path_str = model_path;int FileLoadFlag = initialize_handler(model_path_str, 0);if (FileLoadFlag >= 0 ){return 0;}return FileLoadFlag;
}
template<typename T> std::vector<float> softmax(const T *logits, unsigned int _size, unsigned int &max_id)
{//types::__assert_type<T>();if (_size == 0 || logits == nullptr) return{};float max_prob = 0.f, total_exp = 0.f;std::vector<float> softmax_probs(_size);for (unsigned int i = 0; i < _size; ++i){softmax_probs[i] = std::exp((float)logits[i]);total_exp += softmax_probs[i];}for (unsigned int i = 0; i < _size; ++i){softmax_probs[i] = softmax_probs[i] / total_exp;if (softmax_probs[i] > max_prob){max_id = i;max_prob = softmax_probs[i];}}return softmax_probs;
}
float SqueezeNetLiveSpoof::detect_handler(const unsigned char* pData, int width, int height, int nchannel, int mod)
{if (!pData || width <= 0 || height <= 0) return 0.0f;try {// 1. 将输入数据转换为OpenCV Matcv::Mat input_mat(height, width, nchannel == 3 ? CV_8UC3 : CV_8UC1, (void*)pData);if (nchannel == 1) {cv::cvtColor(input_mat, input_mat, cv::COLOR_GRAY2BGR);}// 2. 预处理图像this->transform(input_mat);// 3. 运行推理mnn_interpreter->runSession(mnn_session);// 4. 获取输出auto output_tensor = mnn_interpreter->getSessionOutput(mnn_session, nullptr);MNN::Tensor host_tensor(output_tensor, output_tensor->getDimensionType());output_tensor->copyToHostTensor(&host_tensor);auto embedding_dims = host_tensor.shape(); // (1,128)const unsigned int hidden_dim = embedding_dims.at(1);const float* embedding_values = host_tensor.host<float>();unsigned int pred_live = 0;auto softmax_probs = softmax<float>(embedding_values, hidden_dim, pred_live);//std::cout << "softmax_probs: " << softmax_probs[0]<<"  "<<softmax_probs[1] << std::endl;float live_score = softmax_probs[0]; // 取真脸概率作为活体分数std::cout << "live_score: " << live_score<< "   spoof_score:"<< softmax_probs[1]<< std::endl;return live_score;} catch (const std::exception& e) {std::cerr << "detect_handler exception: " << e.what() << std::endl;return 0.0f;}
}
void SqueezeNetLiveSpoof::initialize_pretreat() {// 初始化预处理参数MNN::CV::Matrix trans;trans.setScale(1.0f, 1.0f);MNN::CV::ImageProcess::Config img_config;img_config.filterType = MNN::CV::BICUBIC;::memcpy(img_config.mean, meanVals_, sizeof(meanVals_));::memcpy(img_config.normal, normVals_, sizeof(normVals_));img_config.sourceFormat = MNN::CV::BGR;img_config.destFormat = MNN::CV::RGB;pretreat = std::shared_ptr<MNN::CV::ImageProcess>(MNN::CV::ImageProcess::create(img_config));pretreat->setMatrix(trans);
}
void SqueezeNetLiveSpoof::transform(const cv::Mat &mat){cv::Mat mat_rs;cv::resize(mat, mat_rs, cv::Size(input_width, input_height));pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor);
}

四、结果展示

在返回的分类结果中，我们用0.8作为阈值对活体分数进行过滤，得到的结果如下：

五、留下来的问题

一个从数据整理到最后的MNN推理的2D活体检测的工作简单的完结了，这个系列的内容主要目的是讲诉一个模型如何从设计到部署的全过程，过程中的有些描述和个人理解并不一定正确，如果有其他理解或者错处指出，请严重指出。
深度学习之用CelebA_Spoof数据集搭建一个活体检测-数据处理
深度学习之用CelebA_Spoof数据集搭建一个活体检测-模型搭建和训练
深度学习之用CelebA_Spoof数据集搭建一个活体检测-模型验证与测试
深度学习之用CelebA_Spoof数据集搭建一个活体检测-一些模型训练中的改动带来的改善
那么这个系列完结，留下什么问题：
1 2D活体检测有没有更好的方法？
2 训练的过程如何更好更快的调参以及收敛，以及如何寻找更好的特征？
3 在实际使用过程中，怎样提高功能的体验感，至于那些判断错误的，该如何进行处理？
4 如何在不同环境下，保证活体的准确率？
这都是在这个工作中需要重视的，而且这项工作并不会因为有了部署成功就能成功，而是需要不断改善。如果有好的方法和建议，烦请留言告知，我们一起讨论！