【5G核心网】free5GC UPF源码分析

   Usage: ./free5gc-upfd [-f CONFIG_PATH]

1. UPF 启动初始化

    upf_init.c 定义了一堆 UpfOps 列表，包括：

名字	Handler
Library - Bufblk Pool	BufblkPoolInit
Library - Thread	ThreadInit
Library - Timer Pool	TimerPoolInit
Library - Socket Pool	SockPoolInit
Library - GTPv1 Device Pool	Gtpv1DevPoolInit
UPF - Context	UpfContextInit
UPF - Signal Registration	SignalRegister
UPF - Epoll	EpollInit
UPF - Event Queue	EventQueueInit
UPF - Thread	PacketRecvThreadInit
UPF - GTP-U Server	GTPv1ServerInit
UPF - PFCP	PfcpInit
UPF - Routing Setting  UpRouteInit	UpRouteInit
UPF - Buffer Server	BufferServerInit

   1.1 BufblkPoolInit 函数

     缓冲池初始化，使用的是 uint8_t 数组，实现文件 upf/lib/utlt/src/utlt_buff.c

typedef uint8_t bufPool64_t[SIZE_OF_BUF_64 + SIZE_OF_BUF_RESERVED];
typedef uint8_t bufPool128_t[SIZE_OF_BUF_128 + SIZE_OF_BUF_RESERVED];

   1.2 ThreadInit 函数

     线程池 threadPool 初始化 128 ，实现文件 upf/lib/utlt/src/utlt_thread.c

Status ThreadInit() {
    PoolInit(&threadPool, MAX_NUM_OF_THREAD);

    memset(&threadStopCheck, 0, sizeof(threadStopCheck));

    // semaphore create
    sem_t *sem = NULL;
    sem = sem_open("semThreadInit", O_CREAT | O_EXCL, 0644, 0);
    UTLT_Assert(sem != SEM_FAILED, return STATUS_ERROR, "Thread initialize sem_open error");
    sem_unlink("semThreadInit");
    threadStopCheck.semaphore = sem;

    return STATUS_OK;
}

   1.3 UpfContextInit 函数

    UpfContext 结构体定义在文件 upf/src/upf_context.h，static UpfContext self; 

    实例化 UpfContext 进行初始化

   1.4 EpollInit 函数

    调用 epoll_create(MAX_NUM_OF_EVENT)，生成一个 epoll 专用的文件描述符。其实是在内核申请一空间，用来存放想关注的 socket fd 上是否发生以及发生了事件。MAX_NUM_OF_EVENT 就是在这个epoll fd上能关注的最大socket fd数

static Status EpollInit(void *data) {
    UTLT_Assert((Self()->epfd = EpollCreate()) >= 0,
        return STATUS_ERROR, "");
    
    return STATUS_OK;
}

   1.5 GTPv1ServerInit 函数

    Rtnetlink 允许对内核路由表进行读和更改，它用于内核与各个子系统之间（路由子系统、IP地址、链接参数等）的通信，用户空间可以通过NET_LINK_ROUTER socket 与内核进行通信，该过程基于标准的netlink消息进行

      -->  GtpDevListCreate

            -->  GtpTunCreate

                     -->  GtpLinkCreate

                               -->  UdpServerCreate

                                        -->  UdpSockCreate 

                                              -->  SockCreate 创建 sock，从 sock 池分配并调用 socket 创建

                                        -->  UdpSockSetAddr

                                              地址族分为 AF_INET AF_INET6 AF_UNIX

                                        -->  SockSetOpt

                                        -->  SockBind

                               -->  gtp_dev_create 核心

                     -->   SockRegister  sock handler 和 data 赋值

            -->  GtpEpollRegister

    1.5.1 gtp_dev_create 函数

     netlink 头格式

struct nlmsghdr
{
    __u32 nlmsg_len; /* Length of message including header */
    __u16 nlmsg_type; /* Message content */
    __u16 nlmsg_flags; /* Additional flags */
    __u32 nlmsg_seq; /* Sequence number */
    __u32 nlmsg_pid; /* Sending process PID */
};

消息头中各成员属性的解释及说明：

  (1) nlmsg_len：整个消息的长度，按字节计算。包括了Netlink消息头本身。

  (2) nlmsg_type：消息的类型，即是数据还是控制消息。目前(内核版本2.6.21)Netlink仅支持四种类型的控制消息，如下：

       a) NLMSG_NOOP-空消息，什么也不做；

       b) NLMSG_ERROR-指明该消息中包含一个错误；

       c) NLMSG_DONE-如果内核通过Netlink队列返回了多个消息，那么队列的最后一条消息的类型为NLMSG_DONE，其余所有消息的nlmsg_flags属性都被设置NLM_F_MULTI位有效。

       d) NLMSG_OVERRUN-暂时没用到。

   (3) nlmsg_flags：附加在消息上的额外说明信息，如上面提到的NLM_F_MULTI。 

static int _gtp_dev_create(int dest_ns, const char *gtp_ifname, int fd,
			   enum ifla_gtp5g_role role)
{
	char buf[MNL_SOCKET_BUFFER_SIZE];
	struct nlmsghdr *nlh;
	struct ifinfomsg *ifm;
	unsigned int seq = time(NULL);
	struct nlattr *nest, *nest2;

 struct ifinfomsg
 {
    unsigned char  ifi_family;  /* AF_UNSPEC */
    unsigned char  __ifi_pad;   /* unused */
    unsigned short ifi_type;    /* Device type */
    int   ifi_index;   /* Interface index   */
    unsigned int  ifi_flags;   /* Device flags  */
    unsigned int  ifi_change;  /* change mask */
 } 
    其中 ifi_change是为将来预留的，总是被设为 0xFFFFFFFF
       rta_type     value type       description
       --------------------------------------------------------------
       IFLA_UNSPEC    -        unspecified.
       IFLA_ADDRESS    hardware address   interface L2 address
       IFLA_BROADCAST    hardware address   L2 broadcast address.
       IFLA_IFNAME    asciiz string      Device name.
       IFLA_MTU     unsigned int       MTU of the device.
       IFLA_LINK     int        Link type.
       IFLA_QDISC    asciiz string      Queueing discipline.
       IFLA_STATS    struct       Interface Statistics.
       net_device_stats

    1.5.1.2 gtp_put_nlmsg

      RTM_NEWLINK, NLM_F_CREATE | NLM_F_EXCL | NLM_F_ACK

      创建或者删除一个特定的网络接口，或者从一个特定的网络接口上获得信息。这些消息含有一个ifinfomsg类型的结构，紧跟在后面的是一系列的rtattr结构。RTM_NEWLINK 创建

  struct nlmsghdr* mnl_nlmsg_put_header(void * buf)

     保留并为 Netlink 头准备空间

    1.5.1.3 mnl_nlmsg_put_extra_header

    void* mnl_nlmsg_put_extra_header ( struct nlmsghdr * nlh, size_t size )

    Parameters

nlh	pointer to Netlink header
size	size of the extra header that we want to put

    在 mnl_nlmsg_put_header 之后调用

   1.5.2 gtp_dev_talk 函数

    rtnl_open 创建 NETLINK_ROUTE socket，绑定 MNL_SOCKET_AUTOPID

    rtnl_talk 发送以及接收，mnl_cb_run 函数回调运行队列

int mnl_cb_run  (  const void *   buf,
  size_t     numbytes,
  unsigned int   seq,
  unsigned int   portid,
  mnl_cb_t   cb_data,
  void *     data 
)  

  Parameters

buf	buffer that contains the netlink messages
numbytes	number of bytes stored in the buffer
seq	sequence number that we expect to receive
portid	Netlink PortID that we expect to receive
cb_data	callback handler for data messages
data	pointer to data that will be passed to the data callback handler

   1.6 PfcpInit

    1.6.1 PfcpServerInit 函数

    PfcpServer 函数 UdpServerCreate 创建 UDP 服务，8805 端口，SockRegister 将 handler（_pfcpReceiveCB） 进行注册到 sock，_pfcpReceiveCB 函数设置收到的包类型为 UPF_EVENT_N4_MESSAGE，丢入队列进行处理

    1.6.2 PfcpXactInit 函数

    主要结构体 PfcpXact

typedef struct _PfcpXact {
    ListNode    node;
    uint32_t    index;

#define PFCP_LOCAL_ORIGINATOR  0
#define PFCP_REMOTE_ORIGINATOR 1
    uint8_t     origin;
    uint32_t    transactionId;
    PfcpNode    *gnode;

    int         step;               // 1: Init, 2: Trigger, 3: Trigger Reply
    struct {
        uint8_t type;
        Bufblk  *bufBlk;
    } seq[3];

    TimerBlkID  timerResponse;
    uint8_t     responseReCount;
    TimerBlkID  timerHolding;
    uint8_t     holdingReCount;

    struct _PfcpXact    *associatedXact;

#define PfcpXactStoreSession(xact, session) \
    do { \
        UTLT_Assert((xact), break, "xact error"); \
        UTLT_Assert((session), break, "session error"); \
        ((xact)->session) = (session); \
    } while(0)

    void        *session;
    void        *gtpXact;
    Bufblk      *gtpBuf;
} PfcpXact;

   1.7 UpRouteInit 函数

    KnetAddRoute 添加 gtp 设备的路由

# ip route
default via 10.200.200.254 dev eth0 
10.200.200.0/24 dev eth0 proto kernel scope link src 10.200.200.101 
60.60.0.0/24 dev upfgtp0 proto static 

Status UpRouteInit() {
    Status status;

    for (ApnNode *it = ListFirst(&Self()->apnList); it != NULL; it = ListNext(it)) {
        // TODO: only get the first dev here, there will be only one gtp tun dev in the future
        Gtpv1TunDevNode *gtpTunDev = ListFirst(&Self()->gtpv1DevList);

        status = KnetAddRoute(gtpTunDev->ifname, it->subnetIP, it->subnetPrefix, NULL, 0);
        UTLT_Assert(status == STATUS_OK, return STATUS_ERROR, "");
    }

2. UPF 处理 PFCP 关联请求

Status UpfN4HandleAssociationSetupRequest(PfcpXact *xact,
                                          PFCPAssociationSetupRequest *request) {
    PfcpNodeId *nodeId;

    UTLT_Assert(xact, return STATUS_ERROR, "xact error");
    UTLT_Assert(xact->gnode, return STATUS_ERROR,
                "gNode of xact error");
    UTLT_Assert(request->nodeID.presence, return STATUS_ERROR,
                "Request missing nodeId");

      接受 PFCP Association Request，设置状态值为：PFCP_NODE_ST_ASSOCIATED，表示完成状体

/* Accept */
xact->gnode->state = PFCP_NODE_ST_ASSOCIATED;

3. UPF 处理 PFCP 建立请求

    消息类型为： PFCP_SESSION_ESTABLISHMENT_REQUEST，body 结构为： PFCPSessionEstablishmentRequest，处理函数为 UpfN4HandleSessionEstablishmentRequest

Status UpfN4HandleSessionEstablishmentRequest(UpfSession *session, PfcpXact *pfcpXact,
                                              PFCPSessionEstablishmentRequest *request) {
    Status status;
    uint8_t cause = PFCP_CAUSE_REQUEST_ACCEPTED;

    UTLT_Assert(session, return STATUS_ERROR, "Upf Session error");
    UTLT_Assert(pfcpXact, return STATUS_ERROR, "pfcpXact error");

   3.1 UpfN4HandleCreateFar 处理 FAR 转发行为规则

    gtp5g_far_alloc 实例化 gtp5g_far 对象

struct gtp5g_far {
    struct hlist_node    hlist_id;

    u32 id;

//    u8 dest_iface;
    u8 action;                              // apply action

    struct forwarding_parameter *fwd_param;

    struct net_device   *dev;
    struct rcu_head     rcu_head;
};

    CreateFAR 结构体

typedef struct _CreateFAR {
    unsigned long presence;
    FARID fARID;
    ApplyAction applyAction;
    ForwardingParameters forwardingParameters;
    DuplicatingParameters duplicatingParameters;
    BARID bARID;
} __attribute__((packed)) CreateFAR;

   根据 3GPP 定义的 Create FAR IE within PFCP Session Establishment Request

FAR ID	M	This IE shall uniquely identify the FAR among all the FARs configured for that PFCP session.
Apply Action	M	This IE shall indicate the action to apply to the packets, See clauses 5.2.1 and 5.2.3.
Forwarding Parameters	C	This IE shall be present when the Apply Action requests the packets to be forwarded. It may be present otherwise. When present, this IE shall contain the forwarding instructions to be applied by the UP function when the Apply Action requests the packets to be forwarded. See table 7.5.2.3-2.
Duplicating Parameters	C	This IE shall be present when the Apply Action requests the packets to be duplicated. It may be present otherwise. When present, this IE shall contain the forwarding instructions to be applied by the UP function for the traffic to be duplicated, when the Apply Action requests the packets to be duplicated. Several IEs with the same IE type may be present to represent to duplicate the packets to different destinations. See NOTE 1. See table 7.5.2.3-3.
BAR ID	O	When present, this IE shall contain the BAR ID of the BAR defining the buffering instructions to be applied by the UP function when the Apply Action requests the packets to be buffered.

     3.1.1  _pushFarToKernel

          -->  GtpTunnelAddFar

                 -->  NetlinkSockOpen

                        -->  gtp5g_add_far

                               -->  genl_nlmsg_build_hdr

                               -->  gtp5g_build_far_payload

                               -->  genl_socket_talk

   struct nlmsghdr

    Netlink 的报文由消息头和消息体构成，struct nlmsghdr 为消息头：

struct nlmsghdr

{

    __u32 nlmsg_len; /* Length of message including header */

    __u16 nlmsg_type; /* Message content */

    __u16 nlmsg_flags; /* Additional flags */

    __u32 nlmsg_seq; /* Sequence number */

    __u32 nlmsg_pid; /* Sending process PID */

};

   -  nlmsg_len：消息的长度，按字节计算。包括了Netlink消息头

   -  nlmsg_type：消息类型，数据/控制消息。如下：

        a) NLMSG_NOOP，空消息

        b) NLMSG_ERROR，指明该消息中包含一个错误

        c) NLMSG_DONE，如果内核通过Netlink队列返回了多个消息，那么队列的最后一条消息的类型为NLMSG_DONE，其余所有消息的 nlmsg_flags 属性都被设置NLM_F_MULTI位有效。

        d) NLMSG_OVERRUN，暂时没用到

   -  nlmsg_flags：附加在消息上的额外说明信息，如上面提到的 NLM_F_MULTI

   3.2 UpfN4HandleCreatePdr 处理 PDR 包检测规则

    gtp5g_pdr_alloc 实例化 gtp5g_pdr 对象，

struct gtp5g_pdr {
    uint16_t id;
    uint32_t *precedence;
    struct gtp5g_pdi *pdi;

    uint8_t *outer_hdr_removal;
    uint32_t *far_id;

    /* Not in 3GPP spec, just used for routing */
    struct in_addr *role_addr_ipv4;

    /* Not in 3GPP spec, just used for buffering */
    char *unix_sock_path;
};

    _CreatePDR 结构体

typedef struct _CreatePDR {
    unsigned long presence;
    PacketDetectionRuleID pDRID;
    Precedence precedence;
    PDI pDI;
    OuterHeaderRemoval outerHeaderRemoval;
    FARID fARID;
    URRID uRRID;
    QERID qERID;
    ActivatePredefinedRules activatePredefinedRules;
} __attribute__((packed)) CreatePDR;

    PDI SDF 过滤器 很多未实现

   3.3 UpfN4BuildSessionEstablishmentResponse 函数

    响应请求设置类型为 PFCP_SESSION_ESTABLISHMENT_RESPONSE，设置 seid。body 结构体为 pFCPSessionEstablishmentResponse

参考：

   1. int socket(int domain, int type, int protocol);

         参数      描述
      domain    设置通信域(本地(PF_LOCAL),ipv4(AF_INET),ipv6()等)
           type    设置套接字通信类型(TCP,双向字节流(SOCK_STREAM),UDP(SOCK_DGRAM))
     protocol    某个协议的特定类型,既type类型中某个类型.通常只有一个类型,所以设置成0

   2. Netlink

    Netlink有什么优势呢？

    用户空间和内核空间的通信方式有三种：/proc、ioctl、Netlink。前两种都是单向的，Netlink 可以实现双工通信