项目需要在Android平台用MNN部署语音模型,实现实时语音转文字等功能,先简单搭建下环境,跑通直线检测模型.
这边记录下过程,及出现的问题.
项目环境:
- AndroidStudio:2021.1.1 Patch 3
- Gradle:7.2
效果
1.编译MNN生成动/静态库(.so/.a)
上篇已完成库文件编译,可参考Android编译MNN
2.配置库文件和头文件
- 库文件目录:src/main/jniLibs,没有对应目录,自行创建
- 头文件目录:src/main/cpp,没有对应目录,自行创建
3.配置CMake(CMakeLists.txt)
CMakeLists配置字段具体含义可参考Android NDK项目CMakeLists.txt配置导入第三方库
# For more information about using CMake with Android Studio, read the
# documentation: https://d.android.com/studio/projects/add-native-code.html
# Sets the minimum version of CMake required to build the native library.
cmake_minimum_required(VERSION 3.18.1)
# Declares and names the project.
project("mnnmyh")
aux_source_directory(. SRC_LIST) # 当前文件夹下的所有文件，不包含子文件夹
add_library( # Sets the name of the library.
mnnmyh
# Sets the library as a shared library.
SHARED
# Provides a relative path to your source file(s).
#native-lib.cpp)
${SRC_LIST})
#mnn
include_directories(${CMAKE_SOURCE_DIR}/MNN/)
include_directories(${CMAKE_SOURCE_DIR}/MNN/plugin)
add_library(libMNN SHARED IMPORTED)
set_target_properties(
libMNN
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libMNN.so
)
include_directories(${CMAKE_SOURCE_DIR}/MNN/expr)
add_library(libMNN_Express SHARED IMPORTED)
set_target_properties(
libMNN_Express
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libMNN_Express.so
)
find_library( # Sets the name of the path variable.
log-lib
# Specifies the name of the NDK library that
# you want CMake to locate.
log)
target_link_libraries( # Specifies the target library.
mnnmyh
# Links the target library to the log library
# included in the NDK.
${log-lib}
libMNN_Express
libMNN
)
4.build.gradle配置(app/build.gradle)
android {
compileSdk 31
defaultConfig {
...省略无关代码
externalNativeBuild {
cmake {
cppFlags '-std=c++11'
arguments "-DANDROID_STL=c++_shared"
}
}
ndk {
//编译那些cpu架构的库
abiFilters 'armeabi-v7a','arm64-v8a'
}
}
//配置CMakeLists路径,如果创建的是支持C++的项目,此处会自动配置
externalNativeBuild {
cmake {
path file('src/main/cpp/CMakeLists.txt')
version '3.18.1'
}
}
sourceSets {
main {
jniLibs.srcDirs = ['libs']
}
}
//此处可指定NDK版本(AS已下载的版本)
ndkVersion '22.0.7026061'
}
...
5.测试MNN加载调用模型
这边主要参考直线检测模型
- 模型文件放在src/mainn/assets,没有对应目录自行创建.
- 模型加载推理主要逻辑在lane.cpp,lane.hpp
lane.hpp
#ifndef __LANE_H__
#define __LANE_H__
#include <iostream>
#include <cmath>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include "MNN/Interpreter.hpp"
#include "MNN/ImageProcess.hpp"
#include <android/log.h>
#ifndef LOG_TAG
#define LOG_TAG "WZT_MNN"
#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG,LOG_TAG ,__VA_ARGS__) // 定义LOGD类型
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG ,__VA_ARGS__) // 定义LOGI类型
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN,LOG_TAG ,__VA_ARGS__) // 定义LOGW类型
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG ,__VA_ARGS__) // 定义LOGE类型
#define LOGF(...) __android_log_print(ANDROID_LOG_FATAL,LOG_TAG ,__VA_ARGS__) // 定义LOGF类型
#endif
class LaneDetect
{
public:
typedef struct Lanes
{
float x1; // 起点
float y1;
float x2; // 终点
float y2;
float lens;
float conf;
}Lanes;
public:
LaneDetect(const std::string &mnn_path, bool useGPU);
~LaneDetect();
std::vector<Lanes> detect(const cv::Mat& img,unsigned char* image_bytes, int width, int height, double threshold, double lens_threshold);
std::vector<Lanes> decodeHeatmap(const float* heatmap, int width, int height, double threshold, double lens_threshold);
inline int clip(float value);
private:
std::shared_ptr<MNN::Interpreter> m_net = nullptr;
MNN::ScheduleConfig m_config;
MNN::Session *m_session = nullptr;
MNN::Tensor *m_inTensor = nullptr;
std::shared_ptr<MNN::CV::ImageProcess> pretreat = nullptr;
MNN::BackendConfig m_backend_config;
const float m_mean_vals[3] = {
127.5f, 127.5f, 127.5f };
const float m_norm_vals[3] = {
1/127.5f, 1/127.5f, 1/127.5f };
int m_input_size = 512; // 输入尺寸
int m_hm_size = 256; // 特征图大小
int m_top_k = 200; // 取200条线
public:
static LaneDetect *detector;
static bool hasGPU;
static bool toUseGPU;
};
#endif
lane.cpp
#include "lane.hpp"
bool LaneDetect::hasGPU = false;
bool LaneDetect::toUseGPU = false;
LaneDetect *LaneDetect::detector = nullptr;
LaneDetect::LaneDetect(const std::string &mnn_path, bool useGPU)
{
toUseGPU = hasGPU && useGPU;
m_net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(mnn_path.c_str()));
m_backend_config.precision = MNN::BackendConfig::PrecisionMode::Precision_Low; // 精度
m_backend_config.power = MNN::BackendConfig::Power_Normal; // 功耗
m_backend_config.memory = MNN::BackendConfig::Memory_Normal; // 内存占用
m_config.backendConfig = &m_backend_config;
m_config.numThread = 4;
if (useGPU) {
m_config.type = MNN_FORWARD_OPENCL;
}
m_config.backupType = MNN_FORWARD_CPU;
MNN::CV::ImageProcess::Config img_config; // 图像处理
::memcpy(img_config.mean, m_mean_vals, sizeof(m_mean_vals)); // (img - mean)*norm
::memcpy(img_config.normal, m_norm_vals, sizeof(m_norm_vals));
img_config.sourceFormat = MNN::CV::BGR;
img_config.destFormat = MNN::CV::RGB;
pretreat = std::shared_ptr<MNN::CV::ImageProcess>(MNN::CV::ImageProcess::create(img_config));
MNN::CV::Matrix trans;
trans.setScale(1.0f, 1.0f); // scale
pretreat->setMatrix(trans);
m_session = m_net->createSession(m_config); //创建session
m_inTensor = m_net->getSessionInput(m_session, NULL);
m_net->resizeTensor(m_inTensor, {
1, 3, m_input_size, m_input_size});
m_net->resizeSession(m_session);
std::cout << "session created" << std::endl;
}
LaneDetect::~LaneDetect()
{
m_net->releaseModel();
m_net->releaseSession(m_session);
}
inline int LaneDetect::clip(float value)
{
if (value > 0 && value < m_input_size)
return int(value);
else if (value < 0)
return 1;
else
return m_input_size - 1;
}
std::vector<LaneDetect::Lanes> LaneDetect::decodeHeatmap(const float* hm,int w, int h, double threshold, double lens_threshold)
{
// 线段中心点(256*256),线段偏移(4*256*256)
const float* displacement = hm+m_hm_size*m_hm_size;
// exp(center,center);
std::vector<float> center;
for (int i = 0;i < m_hm_size*m_hm_size; i++)
{
center.push_back( hm[i] ); // mlsd.mnn原始需要1/(exp(-hm[i]) + 1)
}
center.resize(m_hm_size*m_hm_size);
std::vector<int> index(center.size(), 0);
for (int i = 0 ; i != index.size() ; i++) {
index[i] = i;
}
sort(index.begin(), index.end(),
[&](const int& a, const int& b) {
return (center[a] > center[b]); // 从大到小排序
}
);
std::vector<Lanes> lanes;
for (int i = 0; i < index.size(); i++)
{
int yy = index[i]/m_hm_size; // 除以宽得行号
int xx = index[i]%m_hm_size; // 取余宽得列号
Lanes Lane;
Lane.x1 = xx + displacement[index[i] + 0*m_hm_size*m_hm_size];
Lane.y1 = yy + displacement[index[i] + 1*m_hm_size*m_hm_size];
Lane.x2 = xx + displacement[index[i] + 2*m_hm_size*m_hm_size];
Lane.y2 = yy + displacement[index[i] + 3*m_hm_size*m_hm_size];
Lane.lens = sqrt(pow(Lane.x1 - Lane.x2,2) + pow(Lane.y1 - Lane.y2,2));
Lane.conf = center[index[i]];
if (center[index[i]] > threshold && lanes.size() < m_top_k)
{
if ( Lane.lens > lens_threshold)
{
Lane.x1 = clip(w * Lane.x1 / (m_input_size / 2));
Lane.x2 = clip(w * Lane.x2 / (m_input_size / 2));
Lane.y1 = clip(h * Lane.y1 / (m_input_size / 2));
Lane.y2 = clip(h * Lane.y2 / (m_input_size / 2));
lanes.push_back(Lane);
}
}
else
break;
}
return lanes;
}
std::vector<LaneDetect::Lanes> LaneDetect::detect(const cv::Mat& img, unsigned char* image_bytes, int width, int height, double threshold, double lens_threshold)
{
// 图像处理
cv::Mat preImage = img.clone();
cv::resize(preImage,preImage,cv::Size(m_input_size,m_input_size));
pretreat->convert(preImage.data, m_input_size, m_input_size, 0, m_inTensor);
// 推理
m_net->runSession(m_session);
MNN::Tensor *output= m_net->getSessionOutput(m_session, NULL);
MNN::Tensor tensor_scores_host(output, output->getDimensionType());
output->copyToHostTensor(&tensor_scores_host);
auto score = output->host<float>(); // 得到结果指针
std::vector<LaneDetect::Lanes> lanes = decodeHeatmap(score, width, height, threshold,lens_threshold);
return lanes;
}
native-lib.cpp
直线检测只用到了init()和detect(),其他是测试opencv的不用管.
#include <jni.h>
#include <string>
#include "lane.hpp"
extern "C"
JNIEXPORT jstring JNICALL
Java_com_moyihen_mnnmyh_JniBitmapUtil_stringFromJNI(JNIEnv *env, jclass clazz) {
std::string hello = "Hello from C++";
return env->NewStringUTF(hello.c_str());
}
extern "C"
JNIEXPORT void JNICALL
Java_com_moyihen_mnnmyh_JniBitmapUtil_init(JNIEnv *env, jclass clazz, jstring name, jstring path,
jboolean use_gpu) {
if (LaneDetect::detector != nullptr) {
delete LaneDetect::detector;
LaneDetect::detector = nullptr;
}
if (LaneDetect::detector == nullptr) {
const char *pathTemp = env->GetStringUTFChars(path, 0);
std::string modelPath = pathTemp;
if (modelPath.empty()) {
LOGE("model path is null");
return;
}
modelPath = modelPath + env->GetStringUTFChars(name, 0);
LOGI("model path:%s", modelPath.c_str());
LaneDetect::detector = new LaneDetect(modelPath, use_gpu);
}
}
extern "C"
JNIEXPORT jobjectArray JNICALL
Java_com_moyihen_mnnmyh_JniBitmapUtil_detect(JNIEnv *env, jclass clazz, jobject bitmap,
jbyteArray image_bytes, jint width, jint height,
jdouble threshold, jdouble nms_threshold) {
jbyte *imageDate = env->GetByteArrayElements(image_bytes, nullptr);
if (nullptr == imageDate) {
LOGE("input image is null");
return nullptr;
}
// 输入参数bitmap为Bitmap, image_bytes为bitmap的byte[]
AndroidBitmapInfo img_size;
AndroidBitmap_getInfo(env, bitmap, &img_size);
if (img_size.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
LOGE("input image format not support");
return nullptr;
}
if (width == 333)
LOGD("input image format not support");
auto *dataTemp = (unsigned char *) imageDate;
cv::Mat srcMatImg;
cv::Mat tempMat(height, width, CV_8UC4, dataTemp); //TODO 应该是 480*640
cv::cvtColor(tempMat, srcMatImg, CV_RGBA2BGR);
if (srcMatImg.channels() != 3) {
LOGE("input image format channels != 3");
}
auto result = LaneDetect::detector->detect(srcMatImg, (unsigned char *) imageDate, width,
height, threshold,
nms_threshold); // 获得结果，转换为java对应的类型，传给mainactivity使用
srcMatImg.release();
tempMat.release();
env->ReleaseByteArrayElements(image_bytes, imageDate, 0);
auto line_cls = env->FindClass("com/moyihen/mnnmyh/LaneInfo");
auto cid = env->GetMethodID(line_cls, "<init>", "(FFFFFF)V");
jobjectArray ret = env->NewObjectArray(result.size(), line_cls, nullptr);
int i = 0;
for (auto &line:result) {
env->PushLocalFrame(1);
jobject obj = env->NewObject(line_cls, cid, line.x1, line.y1, line.x2, line.y2, line.lens,
line.conf);
obj = env->PopLocalFrame(obj);
env->SetObjectArrayElement(ret, i++, obj);
}
return ret;
}
- native方法接口类
public class JniBitmapUtil {
private static final String TAG = "JniBitmapUtil";
// Used to load the 'mnnmyh' library on application startup.
static {
System.loadLibrary("mnnmyh");
}
public static native String stringFromJNI();
public static native void init(String name,String path,boolean useGPU);
public static native LaneInfo[] detect(Bitmap bitmap,byte[] imageBytes,int width,int height,double threshold, double lens_threshold);
}
- Android调用
//初始化
//保证模型在下面的目录
String path = getExternalFilesDir(null) + File.separator;
String name = "mlsd_with_max_sigmoid.mnn";
JniBitmapUtil.init(name,path,true);
//直线检测
...
mMnnTestBinding.bt.setOnClickListener(v -> {
if (mBitmap == null){
Toast.makeText(this, "图片呢", Toast.LENGTH_SHORT).show();
return;
}
byte[] bitmap = bitmapToByteArray(mBitmap);
LaneInfo[] laneInfos = JniBitmapUtil.detect(mBitmap, bitmap, mBitmap.getWidth(), mBitmap.getHeight(), 0.2, 20);
Bitmap bitmap2 = mBitmap.copy(Bitmap.Config.ARGB_8888, true);
Bitmap bitmap1 = drawBoxRects(bitmap2, laneInfos);
mMnnTestBinding.img.setImageBitmap(bitmap1);
});
...
private Bitmap drawBoxRects(Bitmap mutableBitmap, LaneInfo[] results) {
if (results == null || results.length <= 0) {
return mutableBitmap;
}
Log.i(TAG, "drawBoxRects: size = "+results.length);
Canvas canvas = new Canvas(mutableBitmap);
final Paint boxPaint = new Paint();
boxPaint.setAlpha(200);
boxPaint.setStrokeWidth(2);
for (LaneInfo line : results) {
//boxPaint.setColor(Color.argb(255, 255, 0, 0));
boxPaint.setColor(Color.YELLOW);
boxPaint.setStyle(Paint.Style.FILL); // 设置只绘制图形内容
canvas.drawLine(line.x0,line.y0,line.x1,line.y1,boxPaint);
}
return mutableBitmap;
}
private byte[] bitmapToByteArray(Bitmap bitmap) {
int bytes = bitmap.getByteCount();
ByteBuffer buf = ByteBuffer.allocate(bytes);
bitmap.copyPixelsToBuffer(buf);
return buf.array();
}
demo下载地址(https://download.csdn.net/download/qq_35193677/86497475)
6.其他
后期也测了语音模型，可以正常运行，能实现语音转文字demo。
std::string wav = "/storage/emulated/0/Android/data/com.moyihen.mnnmyh/files/DEV1.wav";
auto feature_config = wenet::InitFeaturePipelineConfigFromFlags();
wenet::WavReader wav_reader(wav);
auto feature_pipeline = std::make_shared<wenet::FeaturePipeline>(*feature_config);
feature_pipeline->AcceptWaveform(std::vector<float>(wav_reader.data(), wav_reader.data() + wav_reader.num_sample()));
int num_frames = feature_pipeline->num_frames();
LOGD("音频的frame数 = %d",num_frames);
LOGD("测试音频！ = %d",wav.c_str());
/*std::cout<<"音频的frame数： "<<num_frames<<std::endl;
std::cout<<"测试音频！ "<< wav << std::endl;*/
//for(int i=0;i<10;i++){std::cout<<i<<std::endl;}
wenet::MNNDecoder decoder(feature_pipeline); //, *decode_config);
decoder.Reset();
decoder.print_elements();
decoder.Decode();
std::string result = decoder.get_result();
LOGD("最终解码结果 = %s",result.c_str());
