文章目录
下载源码
git clone https://github.com/google/mediapipe.git
安装Bazelisk
./bazel-5.2.0-installer-linux-x86_64.sh --user
设置环境变量
export PATH="$PATH:$HOME/bin"
编译并运行face_mesh样例CPU
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/face_mesh:face_mesh_cpu
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt
编译并运行face_mesh样例GPU
bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 mediapipe/examples/desktop/face_mesh:face_mesh_gpu
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt --input_video_path=testdir/test2.mp4 --output_video_path=testdir/test2out.mp4
编译成3D人脸特征点的动态库
配置face_detection
/home/liangbaikai/mediapipe/mediapipe/modules/face_detection/face_detection_short_range.pbtxt
下修改
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
修改landmark 模型路径
/home/liangbaikai/mediapipe/mediapipe/modules/face_landmark/face_landmarks_model_loader.pbtxt
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:with_attention"
output_side_packet: "PACKET:model_path"
options: {
[mediapipe.SwitchContainerOptions.ext] {
contained_node: {
calculator: "ConstantSidePacketCalculator"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet {
string_value: "mediapipe/modules/face_landmark/face_landmark.tflite"
}
}
}
}
contained_node: {
calculator: "ConstantSidePacketCalculator"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet {
string_value: "/home/liangbaikai/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
}
}
}
}
}
}
}
1.在mediapipe/example/desktop/下新建文件夹myface_mesh_so
2.在myface_mesh_so文件夹新建 BUILD文件和face_landmark_gpu.cpp文件
BUILD文件:
licenses(["notice"])
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
# Linux only
cc_binary(
name = "face_landmark_gpu",
srcs = ["face_landmark_gpu.cpp"],
linkshared = True,
deps = [
"//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_shared_data_internal",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
],
)
cc_binary(
name = "face_landmark_cpu",
srcs = ["face_landmark_cpu.cpp"],
# linkshared = True,
deps = [
"//mediapipe/graphs/face_mesh:desktop_live_calculators",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
],
)
face_landmark_gpu.cpp
#include <cstdlib>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace LIANGBAIKAI_3DFACE_LANDMARKS{
absl::Status Face3D_Landmarks_init_(std::string pbfilepath,
mediapipe::CalculatorGraph &graph,
mediapipe::GlCalculatorHelper &gpu_helper,
std::unique_ptr<mediapipe::OutputStreamPoller> &poller,
std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks)
{
absl::Status run_status;
std::cout << "GetContents ..."<< std::endl;
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(pbfilepath,&calculator_graph_config_contents));
// std::cout << "Get calculator graph config contents: " << calculator_graph_config_contents << std::endl;
mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(calculator_graph_config_contents);
std::cout << "Initialize the calculator graph."<< std::endl;
MP_RETURN_IF_ERROR(graph.Initialize(config));
std::cout << "Initialize the GPU."<< std::endl;
ASSIGN_OR_RETURN(auto gpu_resources, mediapipe::GpuResources::Create());
MP_RETURN_IF_ERROR(graph.SetGpuResources(std::move(gpu_resources)));
gpu_helper.InitializeForTest(graph.GetGpuResources().get());
// 添加视频输出流
std::cout << "Start running the calculator graph." << std::endl;
mediapipe::StatusOrPoller sop = graph.AddOutputStreamPoller("output_video");
assert(sop.ok());
poller = std::make_unique<mediapipe::OutputStreamPoller>(std::move(sop.value()));
// 添加landmarks输出流
mediapipe::StatusOrPoller sop_landmark = graph.AddOutputStreamPoller("multi_face_landmarks");
assert(sop_landmark.ok());
poller_landmarks = std::make_unique<mediapipe::OutputStreamPoller>(std::move(sop_landmark.value()));
std::cout << "StartRun"<< std::endl;
MP_RETURN_IF_ERROR(graph.StartRun({
}));
return absl::OkStatus();
}
absl::Status Face3D_Landmarks_run_(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,
mediapipe::CalculatorGraph &graph,
mediapipe::GlCalculatorHelper &gpu_helper,
std::unique_ptr<mediapipe::OutputStreamPoller> &poller,
std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks)
{
// Wrap Mat into an ImageFrame.
auto input_frame = absl::make_unique<mediapipe::ImageFrame>(mediapipe::ImageFormat::SRGBA, rgba_mat.cols, rgba_mat.rows,mediapipe::ImageFrame::kGlDefaultAlignmentBoundary);
cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
rgba_mat.copyTo(input_frame_mat);
// Prepare and add graph input packet.
size_t frame_timestamp_us =(double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
MP_RETURN_IF_ERROR(gpu_helper.RunInGlContext(
[&input_frame, &frame_timestamp_us, &graph,&gpu_helper]() -> absl::Status
{
// Convert ImageFrame to GpuBuffer.
auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
auto gpu_frame = texture.GetFrame<mediapipe::GpuBuffer>();
glFlush();
texture.Release();
// Send GPU image packet into the graph.
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream("input_video", mediapipe::Adopt(gpu_frame.release()).At(mediapipe::Timestamp(frame_timestamp_us))));
return absl::OkStatus();
}
));
mediapipe::Packet packet;
if (!poller->Next(&packet)){
std::cout << "no output video" << std::endl;
}
mediapipe::Packet packet_landmarks;
if(poller_landmarks->QueueSize() > 0){
if (poller_landmarks->Next(&packet_landmarks))
{
std::vector<mediapipe::NormalizedLandmarkList> output_landmarks = packet_landmarks.Get<std::vector<mediapipe::NormalizedLandmarkList>>();
if(1 == output_landmarks.size()){
mediapipe::NormalizedLandmarkList single_face_LandmarkList = output_landmarks[0];
point_v.clear();
for (int i = 0; i < single_face_LandmarkList.landmark_size(); ++i){
const mediapipe::NormalizedLandmark landmark = single_face_LandmarkList.landmark(i);
// std::cout << "num : " << i << std::endl;
// std::cout << "x : " << landmark.x() * rgba_mat.cols << std::endl;
// std::cout << "y : " << landmark.y() * rgba_mat.cols << std::endl;
// std::cout << "z : " << landmark.z() * rgba_mat.cols << std::endl;
point_v.push_back(cv::Point3f(landmark.x() * rgba_mat.cols,landmark.y() * rgba_mat.cols,landmark.z() * rgba_mat.cols));
}
}
}
}
return absl::OkStatus();
}
absl::Status Face3D_Landmarks_stop_(mediapipe::CalculatorGraph &graph){
MP_RETURN_IF_ERROR(graph.CloseInputStream("input_video"));
return graph.WaitUntilDone();
}
int Face3D_Landmarks_init(std::string pbfilepath,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks){
absl::Status run_status = Face3D_Landmarks_init_(pbfilepath,graph,gpu_helper,poller,poller_landmarks);
if (!run_status.ok()){
LOG(ERROR) << "Failed to init " << run_status.message();
return 1;
}
return 0;
}
int Face3D_Landmarks_run(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks){
absl::Status run_status = Face3D_Landmarks_run_(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);
if (!run_status.ok()){
LOG(ERROR) << "Failed to run " << run_status.message();
return 1;
}
return 0;
}
int Face3D_Landmarks_stop(mediapipe::CalculatorGraph &graph){
absl::Status run_status = Face3D_Landmarks_stop_(graph);
if (!run_status.ok()){
LOG(ERROR) << "Failed to stop " << run_status.message();
return 1;
}
return 0;
}
}
// int main(int argc, char **argv)
// {
// mediapipe::CalculatorGraph graph;
// mediapipe::GlCalculatorHelper gpu_helper;
// std::unique_ptr<mediapipe::OutputStreamPoller> poller;
// std::unique_ptr<mediapipe::OutputStreamPoller> poller_landmarks;
// std::string pbfile = "/home/caijun/mediapipe/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt"; //GPU
// LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_init(pbfile,graph,gpu_helper ,poller,poller_landmarks);
// cv::Mat src = cv::imread("/home/caijun/mediapipe/mediapipe/examples/desktop/myface_mesh_so/test/huge.jpg");
// cv::Mat rgba_mat;
// cv::cvtColor(src, rgba_mat, cv::COLOR_BGR2RGBA);
// std::vector<cv::Point3f> point_v;
// LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_run(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);
// // for(auto &i : point_v){
// // std::cout << i << std::endl;
// // }
// for(unsigned i = 0; i < point_v.size();i ++){
// std::cout << point_v[i].x << std::endl;
// }
// LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_stop(graph);
// return 0;
// }
3.编译
bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 mediapipe/examples/desktop/myface_mesh_so:face_landmark_gpu
成功后会在mediapipe/bazel-bin/mediapipe/examples/desktop/myface_mesh_so/路径下生成动态库文件
4.动态库使用样例
#include<iostream>
#include<string>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include <sys/time.h>
double const lbk_get_time_ms(){
double t;
struct timeval tv;
gettimeofday(&tv, NULL);
t = tv.tv_sec*1000. + tv.tv_usec/1000.;
return t;
}
namespace LIANGBAIKAI_3DFACE_LANDMARKS{
int Face3D_Landmarks_init(std::string pbfilepath,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks);
int Face3D_Landmarks_run(cv::Mat &rgba_mat,std::vector<cv::Point3f> &point_v,mediapipe::CalculatorGraph &graph,mediapipe::GlCalculatorHelper &gpu_helper,std::unique_ptr<mediapipe::OutputStreamPoller> &poller,std::unique_ptr<mediapipe::OutputStreamPoller> &poller_landmarks);
int Face3D_Landmarks_stop(mediapipe::CalculatorGraph &graph);
}
int main(int argc, char **argv)
{
mediapipe::CalculatorGraph graph;
mediapipe::GlCalculatorHelper gpu_helper;
std::unique_ptr<mediapipe::OutputStreamPoller> poller;
std::unique_ptr<mediapipe::OutputStreamPoller> poller_landmarks;
std::string pbfile = "/home/liangbaikai/mediapipe/mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt"; //GPU
LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_init(pbfile,graph,gpu_helper ,poller,poller_landmarks);
cv::Mat src = cv::imread("/home/liangbaikai/mediapipe/mediapipe/examples/desktop/myface_mesh_so/test/huge.jpg");
cv::Mat rgba_mat;
cv::cvtColor(src, rgba_mat, cv::COLOR_BGR2RGBA);
std::vector<cv::Point3f> point_v;
LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_run(rgba_mat,point_v,graph,gpu_helper,poller,poller_landmarks);
for(unsigned i = 0; i < point_v.size();i ++){
std::cout << point_v[i].x << std::endl;
}
LIANGBAIKAI_3DFACE_LANDMARKS::Face3D_Landmarks_stop(graph);
return 0;
}
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
message("-- project name : test")
project(test)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -Wall -fPIC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3 -Wall -fPIC -g")
set(CMAKE_BUILD_TYPE Release)
set(TARGET test)
set(OpenCV_DIR "/usr/local/share/OpenCV")
find_package(OpenCV REQUIRED)
include_directories(
${
PROJECT_SOURCE_DIR}/include
/home/caijun/mediapipe/
/root/anaconda3/pkgs/abseil-cpp-20211102.0-hd4dd3e8_0/include/
/root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/host/bin/
/root/anaconda3/include
/root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_glog_glog/src/
/root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_gflags_gflags/_virtual_includes/gflags/
/root/.cache/bazel/_bazel_root/90baf8ec675cb237b5d9df7abea664c4/execroot/mediapipe/bazel-out/k8-opt/bin/external/com_github_glog_glog/_virtual_includes/default_glog_headers/
)
link_directories(
/home/liangbaikai/mediapipe/bazel-bin/mediapipe/examples/desktop/myface_mesh_so
/root/anaconda3/pkgs/abseil-cpp-20211102.0-hd4dd3e8_0/lib
)
set(SRC_LIST
${
PROJECT_SOURCE_DIR}/src/main.cpp
)
# add_library(${
TARGET} SHARED ${
SRC_LIST})
add_executable(${
TARGET} ${
SRC_LIST}) #test demo
target_link_libraries(${
TARGET}
${
OpenCV_LIBS}
face_landmark_gpu
# face_landmark_cpu
absl_raw_hash_set
)