Mediapipe+Unity3d实现虚拟手

1.首先，在py端按照Mediapipe库

2.重写手部检测类

class HandGestureCapture:
    def __init__(self):
        self.init_video()
        self.init_mediapipe_hands_detector()

    def init_video(self):
        stream_url = 'http://192.168.1.108:4747/video'
        # 创建一个VideoCapture对象，并指定视频流URL
        self.hand_result=None
        self.capture = cv2.VideoCapture(stream_url)

    def init_mediapipe_hands_detector(self):
        hand_options=HandLandmarkerOptions(
            min_hand_detection_confidence=0.8,
            min_hand_presence_confidence=0.8,
            min_tracking_confidence=0.8,
            base_options=python.BaseOptions(model_asset_path='./hand_landmarker.task')
        )

        self.hand_detector=vision.HandLandmarker.create_from_options(hand_options)

    def run(self):
        threading.Thread(target=self.mouse_move,daemon=True).start()
        while True:
            cur_time=self.get_cur_time()
            ret, frame = self.capture.read()
            if ret == True: #有图像输入
                frame_as_numpy_array = numpy.asarray(frame) #格式转换
                mp_image=
                mp.Image(image_format=mp.ImageFormat.SRGB,data=frame_as_numpy_array)
                self.hand_detector.detect_async(mp_image,self.get_cur_time())  #检测
                if self.hand_result:
                   landmarks_data = []
                   for hand_landmarks in 
                        self.hand_result.hand_landmarks:  # 单只手的数据
                        index=0
                        for i in hand_landmarks:  # 每一个关节数据
                            landmark_info = {
                                'id': index, 'x': i.x, 'y': i.y, 'z': i.z
                            }
                            index+=1
                            landmarks_data.append(landmark_info)

每只手指头的关键点信息全部存储在landmarks_data里面了，接下来需要将这些数据传到unity端。采用udp传输的方式，随机挑一个端口号，并将landmarks_data变成json格式的数据进行传输。

message_data = {"landmarks": landmarks_data}
message = json.dumps(message_data)  # 要发送的数据包
try:
    sock.sendto(message.encode(), ('127.0.0.1', 2346))
    print("数据已发送:", message)  # 确认数据已发送
except Exception as e:
    print(f"发送数据时出错: {e}")  # 捕捉并打印错误信息

sendto()函数里面，写上ip与端口号，由于是本机，因此ip写:127.0.0.1就行，端口随便挑一个空的

3.Unity端导入一个手的模型

这个模型的骨骼结构要和mediapipe传过来的数据符合，即要有21个关结点，每只手指头四个外加手腕一个。

实在不符合那就自己添加空物体到结点位置，再为手建立骨骼信息

4.接收UDP数据

这里就是建立udp接收py传来的数据，将json格式转为自定义的landmark数据结构，然后将对应的关节点信息保存到avatarJoint中

public class udp3 : MonoBehaviour
{
    private Vector3[] handJoints; // 存储每个关节点的 Transform
    private UdpClient udpClient;
    public int port = 2346;
    private IPEndPoint endPoint;
    public GameObject[] h;
    public AvatarJoint avatarJoint;
    // JSON 数据结构
    [System.Serializable]
    public class Landmark
    {
        public int index;
        public float x;
        public float y;
        public float z;
    }

    [System.Serializable]
    public class LandmarkWrapper
    {
        public List<Landmark> landmarks;
    }

    void Start()
    {
        // 初始化 UDP 接收
        udpClient = new UdpClient(port);
        handJoints = new Vector3[21];
    }

    void Update()
    {
        // 检查是否有数据可接收
        if (udpClient.Available > 0)
        {
            // 接收并解析 JSON 数据
            byte[] data = udpClient.Receive(ref endPoint);
            string jsonString = Encoding.UTF8.GetString(data);
            Debug.Log("接收到的数据: " + jsonString);  // 打印接收到的数据，便于调试

            try
            {
                LandmarkWrapper wrapper = JsonUtility.FromJson<LandmarkWrapper>(jsonString);
                List<Landmark> landmarks = wrapper.landmarks;

                // 确保接收到的数据和骨骼数量一致
                if (landmarks.Count != 21)
                {
                    Debug.LogError("关节点数量不匹配！");
                    return;
                }

                // 更新 AvatarJoint 的 pose_joint 数组
                for (int i = 0; i < landmarks.Count; i++)
                {
                    Vector3 newPosition = new Vector3(
                        landmarks[i].x * 1, // 转换为 Unity 坐标
                         landmarks[i].y* 1,
                        landmarks[i].z * 1
                    );
                    avatarJoint.pose_joint[i] = newPosition; // 更新 AvatarJoint 数据
                }
            }
            catch (System.Exception e)
            {
                Debug.LogError("JSON 解析出错: " + e.Message);
            }
        }

    }
}

5.驱动手部的运动

原先我是直接将关节点信息映射到unity里面，但这样的方式使得整只手没办法正常显示。后来又采用相对旋转的方式来更新，对每个手指头，采取由根到末的更新，即计算向量差，然后旋转关节点，但效果依然不好。

最后在网上学到一个方法，自建一个类，用树结构来对应父子结点，更新动作从叶子向上更新。

基类:JointBase

using System;
using System.IO;
using UnityEngine;

public class JointBase : MonoBehaviour
{
    [SerializeField]
    [HideInInspector]
    public string path;

    public Vector3[] pose_joint = new Vector3[21];

    public GameObject wrist;         // 手腕

    public GameObject Thumba;        // 大拇指
    public GameObject Thumbb;      
    public GameObject Thumbc;       
    public GameObject Thumbd;

    public GameObject Indexa;        // 食指
    public GameObject Indexb;
    public GameObject Indexc;
    public GameObject Indexd;

    public GameObject Middlea;        // 中指
    public GameObject Middleb;
    public GameObject Middlec;
    public GameObject Middled;

    public GameObject Ringa;        //无名指 
    public GameObject Ringb;
    public GameObject Ringc;
    public GameObject Ringd;

    public GameObject Pinkya;        // 小拇指
    public GameObject Pinkyb;
    public GameObject Pinkyc;
    public GameObject Pinkyd;
    protected virtual float speed { get { return 5f; } }

    protected float lerp = 0f;
    protected Vector3[] skeleton;
    private int idx = 0, max = 100;

    protected void InitData()
    {

    }


    public void Reinit()
    {
        idx = 0;
        lerp = 0;
        InitData();
    }


    protected virtual void Update()
    {
        lerp += speed * Time.deltaTime;
        LerpUpdate(lerp);
        if (lerp >= 1)
        {
            if (++idx >= max) idx = 0;
            Array.Copy(skeleton, idx * 21, pose_joint, 0, 21);
        }
    }


    protected virtual void LerpUpdate(float lerp)
    {

    }

}

继承类：AvatorJoint

using UnityEngine;

public class AvatarJoint : JointBase
{

    public class AvatarTree
    {
        public Transform transf;
        public AvatarTree[] childs;
        public AvatarTree parent;
        public int idx;  // pose_joint's index

        public AvatarTree(Transform tf, int count, int idx, AvatarTree parent = null)
        {
            this.transf = tf;
            this.parent = parent;
            this.idx = idx;
            if (count > 0)
            {
                childs = new AvatarTree[count];
            }
        }

        public Vector3 GetDir()
        {
            if (parent != null)
            {
                return transf.position - parent.transf.position; //计算到父节点位置
            }
            return Vector3.up;
        }
    }

    private AvatarTree tree, Thumb, Index, Middle, Ring, Pinky;
    private AvatarTree Thumb1, Thumb2, Thumb3, Index1, Index2, Index3, Middle1, Middle2, Middle3,Ring1, Ring2, Ring3,Pinky1, Pinky2, Pinky3;

    protected override float speed { get { return 5f; } }

    void Start()
    {
        InitData();
        BuildTree();
    }

    void BuildTree()
    {
        tree = new AvatarTree(wrist.transform, 5, 0);
        Thumb = tree.childs[0] = new AvatarTree(Thumba.transform, 1, 1, tree);
        Index = tree.childs[1] = new AvatarTree(Indexa.transform, 1, 5, tree);
        Middle = tree.childs[2] = new AvatarTree(Middlea.transform, 1, 9, tree);
        Ring = tree.childs[3] = new AvatarTree(Ringa.transform, 1, 13, tree);
        Pinky = tree.childs[4] = new AvatarTree(Pinkya.transform, 1, 17, tree);


        Thumb1 = Thumb.childs[0] = new AvatarTree(Thumbb.transform, 1, 2, Thumb);
        Thumb2 = Thumb1.childs[0] = new AvatarTree(Thumbc.transform, 1, 3, Thumb1);
        Thumb3 = Thumb2.childs[0] = new AvatarTree(Thumbd.transform, 1, 4, Thumb2);

        Index1 = Index.childs[0] = new AvatarTree(Indexb.transform, 1, 6, Index);
        Index2 = Index1.childs[0] = new AvatarTree(Indexc.transform, 1, 7, Index1);
        Index3 = Index2.childs[0] = new AvatarTree(Indexd.transform, 1, 8, Index2);

        Middle1 = Middle.childs[0] = new AvatarTree(Middleb.transform, 1, 10, Middle);
        Middle2 = Middle1.childs[0] = new AvatarTree(Middlec.transform, 1, 11, Middle1);
        Middle3 = Middle2.childs[0] = new AvatarTree(Middled.transform, 1, 12, Middle2);

        Ring1 = Ring.childs[0] = new AvatarTree(Ringb.transform, 1, 14, Ring);
        Ring2 = Ring1.childs[0] = new AvatarTree(Ringc.transform, 1, 15, Ring1);
        Ring3 = Ring2.childs[0] = new AvatarTree(Ringd.transform, 1, 16, Ring2);

        Pinky1 = Pinky.childs[0] = new AvatarTree(Pinkyb.transform, 1, 18, Pinky);
        Pinky2 = Pinky1.childs[0] = new AvatarTree(Pinkyc.transform, 1, 19, Pinky1);
        Pinky3 = Pinky2.childs[0] = new AvatarTree(Pinkyd.transform, 1, 20, Pinky2);
    }  //手动建树


    protected override void LerpUpdate(float lerp)
    {
        UpdateBone(Thumb3, lerp);
        UpdateBone(Index3, lerp);
        UpdateBone(Middle3, lerp);
        UpdateBone(Ring3, lerp);
        UpdateBone(Pinky3, lerp);
        UpdateBone(Thumb2, lerp);
        UpdateBone(Index2, lerp);
        UpdateBone(Middle2, lerp);
        UpdateBone(Ring2, lerp);
        UpdateBone(Pinky2, lerp);
        UpdateBone(Thumb1, lerp);
        UpdateBone(Index1, lerp);
        UpdateBone(Middle1, lerp);
        UpdateBone(Ring1, lerp);
        UpdateBone(Pinky1, lerp);
        UpdateBone(Thumb, lerp);
        UpdateBone(Index, lerp);
        UpdateBone(Middle, lerp);
        UpdateBone(Ring, lerp);
        UpdateBone(Pinky, lerp);
    }


    private void UpdateTree(AvatarTree tree, float lerp) //更新树
    {
        if (tree.parent != null)
        {
            UpdateBone(tree, lerp);
        }
        if (tree.childs != null)
        {
            for (int i = 0; i < tree.childs.Length; i++)
                UpdateTree(tree.childs[i], lerp);
        }
    }

    private void UpdateBone(AvatarTree tree, float lerp)  //更新骨骼位置，子到父位置变化
    {
        var dir1 = tree.GetDir();
        var dir2 = pose_joint[tree.idx] - pose_joint[tree.parent.idx];
        dir2.y = -dir2.y;
        Quaternion rot = Quaternion.FromToRotation(dir1, dir2);
        Quaternion rot1 = tree.parent.transf.rotation;
        tree.parent.transf.rotation = Quaternion.Lerp(rot1, rot * rot1, lerp);
    }

    void Update()
    {
        float lerp = Time.deltaTime * speed; // 动作平滑速度
        LerpUpdate(lerp);
    }


}

将AvatorJoint脚本移到模型的wrist上，并把对应结点拖入脚本的变量中

6.运行

py端启动代码，unity端点击开始游戏：

后续会去学习更精确的控制手及添加一些滤波算法减少手部在光线较差环境的扰动，并会学习制作虚拟人，需要代码的留言即可