在ip_local_deliver中，如果检测到是分片包，则需要将报文进行重组。其所有的分片被重新组合后才能提交到上层协议，每一个被重新组合的数据包文用ipq结构实例来表示
struct ipq { struct inet_frag_queue q; u32 user;//分片来源 __be32 saddr;//原地址 __be32 daddr;//目的地址 __be16 id;//ip报文序列号 u8 protocol;//上层协议号 //这四个字段来自ip首部是为了确定来自哪个ip数据报文 u8 ecn; /* RFC3168 support */ u16 max_df_size; /* largest frag with DF set seen */ int iif; int vif; /* L3 master device index */ unsigned int rid;//已收到的分片计数器 struct inet_peer *peer;//记录发送方信息 //通过rid peer 可以防止Dos攻击 };
网络空间分段管理结构
struct inet_frags { struct inet_frag_bucket hash[INETFRAGS_HASHSZ];//哈希队列 struct work_struct frags_work;//工作队列 unsigned int next_bucket; unsigned long last_rebuild_jiffies; bool rebuild; /* The first call to hashfn is responsible to initialize * rnd. This is best done with net_get_random_once. * * rnd_seqlock is used to let hash insertion detect * when it needs to re-lookup the hash chain to use. */ u32 rnd;//随机数 seqlock_t rnd_seqlock;// int qsize;//队列长度 unsigned int (*hashfn)(const struct inet_frag_queue *); bool (*match)(const struct inet_frag_queue *q, const void *arg);//分段队列匹配函数 void (*constructor)(struct inet_frag_queue *q, const void *arg); void (*destructor)(struct inet_frag_queue *); void (*frag_expire)(unsigned long data);//队列过期处理函数 struct kmem_cache *frags_cachep; const char *frags_cache_name; };
struct netns_frags { /* The percpu_counter "mem" need to be cacheline aligned. * mem.count must not share cacheline with other writers */ struct percpu_counter mem ____cacheline_aligned_in_smp; /* sysctls */ int timeout;超时时间 int high_thresh;内存使用上限 int low_thresh;内存使用下限 int max_dist; };
/** * struct inet_frag_queue - fragment queue * * @lock: spinlock protecting the queue * @timer: queue expiration timer * @list: hash bucket list * @refcnt: reference count of the queue * @fragments: received fragments head * @fragments_tail: received fragments tail * @stamp: timestamp of the last received fragment * @len: total length of the original datagram * @meat: length of received fragments so far * @flags: fragment queue flags * @max_size: maximum received fragment size * @net: namespace that this frag belongs to * @list_evictor: list of queues to forcefully evict (e.g. due to low memory) */ struct inet_frag_queue {//inet分段队列头 spinlock_t lock;smp环境下 需要 struct timer_list timer;队列定时器，组装非常耗时，不能无休止的等待分片的到达 struct hlist_node list;哈希节点，链入inet分段管理结构的哈希队列 atomic_t refcnt;计数器 struct sk_buff *fragments;分段数据包队列 struct sk_buff *fragments_tail; ktime_t stamp;时间戳 int len;数据包结束位置offset+len int meat;与原数据长度的差距，如果和原数据包长度一样代表接收完成 __u8 flags; u16 max_size; struct netns_frags *net;指向网络空寂分段管理结构 struct hlist_node list_evictor; };
1.1、 IP分组的初始化
void __init ipfrag_init(void) { ip4_frags_ctl_register(); register_pernet_subsys(&ip4_frags_ops);//向内核注册ipv4分段管理函数 ip4_frags.hashfn = ip4_hashfn;//设置计算hash的函数 //设置初始化ip 分段队列的构造函数 ip4_frags.constructor = ip4_frag_init; //析构函数 ip4_frags.destructor = ip4_frag_free; //队列机构长度 ip4_frags.qsize = sizeof(struct ipq); //对比ip分段队列hook ip4_frags.match = ip4_frag_match; //设置分段队列过期处理函数 ip4_frags.frag_expire = ip_expire; ip4_frags.frags_cache_name = ip_frag_cache_name; if (inet_frags_init(&ip4_frags)) panic("IP: failed to allocate ip4_frags cache\n"); } int inet_frags_init(struct inet_frags *f) { int i; //初始化工作队列 INIT_WORK(&f->frags_work, inet_frag_worker); for (i = 0; i < INETFRAGS_HASHSZ; i++) { struct inet_frag_bucket *hb = &f->hash[i];//初始化hash 队列头 spin_lock_init(&hb->chain_lock); INIT_HLIST_HEAD(&hb->chain); } seqlock_init(&f->rnd_seqlock); f->last_rebuild_jiffies = 0; f->frags_cachep = kmem_cache_create(f->frags_cache_name, f->qsize, 0, 0, NULL); if (!f->frags_cachep) return -ENOMEM; return 0; } EXPORT_SYMBOL(inet_frags_init);
int ip_local_deliver(struct sk_buff *skb) { /* * Reassemble IP fragments. */ struct net *net = dev_net(skb->dev); /* 分片重组 */ if (ip_is_fragment(ip_hdr(skb))) { if (ip_defrag(net, skb, IP_DEFRAG_LOCAL_DELIVER)) return 0; } /* 经过LOCAL_IN钩子点 */ return NF_HOOK(NFPROTO_IPV4, NF_INET_LOCAL_IN, net, NULL, skb, skb->dev, NULL, ip_local_deliver_finish); }
1.2、 ip分片报文重组的处理
/* Process an incoming IP datagram fragment. */ int ip_defrag(struct net *net, struct sk_buff *skb, u32 user) { struct net_device *dev = skb->dev ? : skb_dst(skb)->dev; int vif = l3mdev_master_ifindex_rcu(dev); struct ipq *qp; //递增计数 __IP_INC_STATS(net, IPSTATS_MIB_REASMREQDS); skb_orphan(skb); /* Lookup (or create) queue header* 查找或创建IP分片队列 */ qp = ip_find(net, ip_hdr(skb), user, vif); if (qp) {/* 分片队列存在 */ int ret; spin_lock(&qp->q.lock); ret = ip_frag_queue(qp, skb);//分片数据包入队重组数据包 spin_unlock(&qp->q.lock); ipq_put(qp); return ret; } /* 创建新的ip分片队列失败，内存不足递增失败计数*/ __IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS); kfree_skb(skb); return -ENOMEM; } EXPORT_SYMBOL(ip_defrag);
1.2.2 ip_find 根据ip首部以及user标志 在ipq散列表中查找对应的ipq。
/* Find the correct entry in the "incomplete datagrams" queue for * this IP datagram, and create new one, if nothing is found. enum ip_defrag_users { IP_DEFRAG_LOCAL_DELIVER, IP_DEFRAG_CALL_RA_CHAIN, IP_DEFRAG_CONNTRACK_IN, __IP_DEFRAG_CONNTRACK_IN_END = IP_DEFRAG_CONNTRACK_IN + USHRT_MAX, IP_DEFRAG_CONNTRACK_OUT, __IP_DEFRAG_CONNTRACK_OUT_END = IP_DEFRAG_CONNTRACK_OUT + USHRT_MAX, IP_DEFRAG_CONNTRACK_BRIDGE_IN, __IP_DEFRAG_CONNTRACK_BRIDGE_IN = IP_DEFRAG_CONNTRACK_BRIDGE_IN + USHRT_MAX, IP_DEFRAG_VS_IN, IP_DEFRAG_VS_OUT, IP_DEFRAG_VS_FWD, IP_DEFRAG_AF_PACKET, IP_DEFRAG_MACVLAN, }; */ static struct ipq *ip_find(struct net *net, struct iphdr *iph, u32 user, int vif) { struct inet_frag_queue *q; struct ip4_create_arg arg; unsigned int hash; /* 记录ip头和输入信息 */ arg.iph = iph; arg.user = user; arg.vif = vif; /* 通过id，源地址，目的地址，协议计算hash */ hash = ipqhashfn(iph->id, iph->saddr, iph->daddr, iph->protocol); /* 根据hash值查找或创建队列 */ q = inet_frag_find(&net->ipv4.frags, &ip4_frags, &arg, hash); if (IS_ERR_OR_NULL(q)) { inet_frag_maybe_warn_overflow(q, pr_fmt()); return NULL; } return container_of(q, struct ipq, q); } struct inet_frag_queue *inet_frag_find(struct netns_frags *nf, struct inet_frags *f, void *key, unsigned int hash) { struct inet_frag_bucket *hb; struct inet_frag_queue *q; int depth = 0; /* 分片内存已经超过了低限 */ if (frag_mem_limit(nf) > nf->low_thresh) /* 进行节点回收 */ inet_frag_schedule_worker(f); //工作队列回调函数为inet_frag_worker hash &= (INETFRAGS_HASHSZ - 1); hb = &f->hash[hash]; /* 找到hash桶 */ spin_lock(&hb->chain_lock); hlist_for_each_entry(q, &hb->chain, list) { /* 遍历链表 */ if (q->net == nf && f->match(q, key)) { atomic_inc(&q->refcnt); /* 增加引用计数 */ spin_unlock(&hb->chain_lock); return q; } depth++;/* 记录查找深度 */ } spin_unlock(&hb->chain_lock); /* 未找到 */ /* 桶节点的链表深度不超过限定 */ if (depth <= INETFRAGS_MAXDEPTH) return inet_frag_create(nf, f, key);/* 创建节点返回 */ if (inet_frag_may_rebuild(f)) { /* 如果已经超过了重建间隔时间，则重建 */ if (!f->rebuild) f->rebuild = true; inet_frag_schedule_worker(f); } return ERR_PTR(-ENOBUFS); } EXPORT_SYMBOL(inet_frag_find);
如果查找不到则会创建一个ipq 并将其插入链表中
static struct inet_frag_queue *inet_frag_create(struct netns_frags *nf, struct inet_frags *f, void *arg) { struct inet_frag_queue *q; q = inet_frag_alloc(nf, f, arg);//分配队列头结构空间 if (!q) return NULL; return inet_frag_intern(nf, q, f, arg); } static struct inet_frag_queue *inet_frag_alloc(struct netns_frags *nf, struct inet_frags *f, void *arg) { struct inet_frag_queue *q; if (frag_mem_limit(nf) > nf->high_thresh) {//内存超过警戒线 回收内存 inet_frag_schedule_worker(f); return NULL; } q = kmem_cache_zalloc(f->frags_cachep, GFP_ATOMIC); if (!q) return NULL; q->net = nf;//记录下网络空间的分段管理结构指针 f->constructor(q, arg);//之前初始化时，构造函数来初始化-ip4_frag_init add_frag_mem_limit(nf, f->qsize);//sum 网络空间的分段内存 setup_timer(&q->timer, f->frag_expire, (unsigned long)q);//定时器initand run spin_lock_init(&q->lock); atomic_set(&q->refcnt, 1); return q; } static void ip4_frag_init(struct inet_frag_queue *q, const void *a) { struct ipq *qp = container_of(q, struct ipq, q);//获取分段队列指针 struct netns_ipv4 *ipv4 = container_of(q->net, struct netns_ipv4, frags); struct net *net = container_of(ipv4, struct net, ipv4); const struct ip4_create_arg *arg = a;//ipv4的分段信息指针 qp->protocol = arg->iph->protocol;//IP层头部协议 qp->id = arg->iph->id;//ip层id qp->ecn = ip4_frag_ecn(arg->iph->tos); qp->saddr = arg->iph->saddr; qp->daddr = arg->iph->daddr; qp->vif = arg->vif; qp->user = arg->user; //记录对方信息 qp->peer = q->net->max_dist ? inet_getpeer_v4(net->ipv4.peers, arg->iph->saddr, arg->vif, 1) : NULL; } static struct inet_frag_queue *inet_frag_intern(struct netns_frags *nf, struct inet_frag_queue *qp_in, struct inet_frags *f, void *arg) { struct inet_frag_bucket *hb = get_frag_bucket_locked(qp_in, f); struct inet_frag_queue *qp; #ifdef CONFIG_SMP /* With SMP race we have to recheck hash table, because * such entry could have been created on other cpu before * we acquired hash bucket lock. */ hlist_for_each_entry(qp, &hb->chain, list) { if (qp->net == nf && f->match(qp, arg)) { atomic_inc(&qp->refcnt); spin_unlock(&hb->chain_lock); qp_in->flags |= INET_FRAG_COMPLETE; inet_frag_put(qp_in, f); return qp; } } #endif qp = qp_in; if (!mod_timer(&qp->timer, jiffies + nf->timeout)) atomic_inc(&qp->refcnt); atomic_inc(&qp->refcnt);//链入inet分段管理结构的hash队列 hlist_add_head(&qp->list, &hb->chain); spin_unlock(&hb->chain_lock); return qp; }
1/2/3 分片数据包加入重组数据包
/* Add new segment to existing queue. */ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) { struct sk_buff *prev, *next; struct net_device *dev; unsigned int fragsize; int flags, offset; int ihl, end; int err = -ENOENT; u8 ecn; if (qp->q.flags & INET_FRAG_COMPLETE) //分段队列接收完成 则释放此分片返回 goto err; /*数据包没有分段标志or 分段队列间隔过大 //重现调整分段队列是否出错 如果不是本地生成的分片，则调用ip_frag_too_far 检测 是否存在 dos攻击，存在攻击则调用邋ip_frag_reinit释放 所用分片 */ if (!(IPCB(skb)->flags & IPSKB_FRAG_COMPLETE) && unlikely(ip_frag_too_far(qp)) && unlikely(err = ip_frag_reinit(qp))) { ipq_kill(qp);//将ipq从散列表中移除停止定时器 计数器减一 // 调用ipq_unlink 设置ipq为complete状态，只有complete状态才能释放 goto err; } ecn = ip4_frag_ecn(ip_hdr(skb)->tos); offset = ntohs(ip_hdr(skb)->frag_off); flags = offset & ~IP_OFFSET; offset &= IP_OFFSET; offset <<= 3; /* offset is in 8-byte chunks */ ihl = ip_hdrlen(skb); /* 获取ip首部中的数据标志位 片的偏移 首部长度 */ /* Determine the position of this fragment. */ end = offset + skb->len - skb_network_offset(skb) - ihl; err = -EINVAL; /**/ /* Is this the final fragment? 如果是最后一个片则先对分片进行检测 */ if ((flags & IP_MF) == 0) { /* If we already have some bits beyond end * or have different end, the segment is corrupted. 结束位置小于前一个位置，ipq已经有 last_in 标志且分片末尾不等于原始数据长度 */ if (end < qp->q.len || ((qp->q.flags & INET_FRAG_LAST_IN) && end != qp->q.len)) goto err; qp->q.flags |= INET_FRAG_LAST_IN; qp->q.len = end; /*通过校验并设置为last_in标志，存储完整的数据长度*/ } else { if (end&7) {//按8字节对其 end &= ~7; if (skb->ip_summed != CHECKSUM_UNNECESSARY) skb->ip_summed = CHECKSUM_NONE; } if (end > qp->q.len) { /* 结束地址大于前一个分段数据地址 Some bits beyond end -> corruption. 如果设置了最后一个分段数据标志 表示最后一个包，则错误*/ if (qp->q.flags & INET_FRAG_LAST_IN) goto err; qp->q.len = end;//记录当前分段数据块的结束位置 } } if (end == offset)//等于起始位置 即分片区数据长度为0 goto err; err = -ENOMEM;//去掉ip首部 if (!pskb_pull(skb, skb_network_offset(skb) + ihl)) goto err; //skb 数据长度为end-offset ip 有效载荷长度 err = pskb_trim_rcsum(skb, end - offset); if (err) goto err; /* Find out which fragments are in front and at the back of us * in the chain of fragments so far. We must know where to put * this fragment, right? */ prev = qp->q.fragments_tail; if (!prev || FRAG_CB(prev)->offset < offset) { next = NULL; goto found; } prev = NULL; for (next = qp->q.fragments; next != NULL; next = next->next) { if (FRAG_CB(next)->offset >= offset) break; /* bingo! */ prev = next; }/*确定分片在链表中的位置，分片到达的时间顺序不同 ipq 上的分片按照分片偏移值大小排序 */ found: /* We found where to put this one. Check for overlap with * preceding fragment, and, if needed, align things so that * any overlaps are eliminated. 检验和和上一个分片数据是否有重叠 */ if (prev) { int i = (FRAG_CB(prev)->offset + prev->len) - offset; if (i > 0) {//有重叠 调用pskb_pull 消除重叠 offset += i; err = -EINVAL; if (end <= offset) goto err; err = -ENOMEM; if (!pskb_pull(skb, i)) goto err; if (skb->ip_summed != CHECKSUM_UNNECESSARY) skb->ip_summed = CHECKSUM_NONE; } } err = -ENOMEM; /*如果和后面一个分片的数据有重叠， 部分重叠还是完全重叠； 重叠部分数据超过下一个分片的数据长度，咋释放 下发一个分片并在检查与后面第二个分片的数据是否 有重叠，如果没有超过下一个则调整下一个分片。 如此反复直到对所有分片都检测完。 调整片的偏移以及分片总长度 */ while (next && FRAG_CB(next)->offset < end) { int i = end - FRAG_CB(next)->offset; /* overlap is 'i' bytes */ if (i < next->len) { /* Eat head of the next overlapped fragment * and leave the loop. The next ones cannot overlap. */ if (!pskb_pull(next, i)) goto err; FRAG_CB(next)->offset += i; qp->q.meat -= i; if (next->ip_summed != CHECKSUM_UNNECESSARY) next->ip_summed = CHECKSUM_NONE; break; } else { struct sk_buff *free_it = next; /* Old fragment is completely overridden with * new one drop it. */ next = next->next; if (prev) prev->next = next; else qp->q.fragments = next; qp->q.meat -= free_it->len; sub_frag_mem_limit(qp->q.net, free_it->truesize); kfree_skb(free_it); } } FRAG_CB(skb)->offset = offset;//当前片的偏移 /* Insert this fragment in the chain of fragments. 当前的片插入到ipq队列中相应的位置*/ skb->next = next; if (!next) qp->q.fragments_tail = skb; if (prev) prev->next = skb; else qp->q.fragments = skb; dev = skb->dev; if (dev) { qp->iif = dev->ifindex; skb->dev = NULL; } qp->q.stamp = skb->tstamp;//更新时间搓 qp->q.meat += skb->len;//sum ipq已收到分片的总长度 qp->ecn |= ecn; //分片组装模块的所占内存的总长度 add_frag_mem_limit(qp->q.net, skb->truesize); if (offset == 0)//为第一个片 设置标志 qp->q.flags |= INET_FRAG_FIRST_IN; fragsize = skb->len + ihl; if (fragsize > qp->q.max_size) qp->q.max_size = fragsize; if (ip_hdr(skb)->frag_off & htons(IP_DF) && fragsize > qp->max_df_size) qp->max_df_size = fragsize; if (qp->q.flags == (INET_FRAG_FIRST_IN | INET_FRAG_LAST_IN) && qp->q.meat == qp->q.len) {//所有报文都到齐则重组 unsigned long orefdst = skb->_skb_refdst; skb->_skb_refdst = 0UL; err = ip_frag_reasm(qp, prev, dev); skb->_skb_refdst = orefdst; return err; } skb_dst_drop(skb); return -EINPROGRESS; err: kfree_skb(skb); return err; }
ip_frag_reasm 重组报文；
* Build a new IP datagram from all its fragments. */ /* *用于组装已到齐的所有分片，当原始 * 数据包的所有分片都已到齐时，会调用此函 * 数组装分片。 */ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *prev, struct net_device *dev) { struct net *net = container_of(qp->q.net, struct net, ipv4.frags); struct iphdr *iph; struct sk_buff *fp, *head = qp->q.fragments; int len; int ihlen; int err; u8 ecn; /* * 要开始组装了，因此调用ipq_kill()将此ipq结点从 * ipq散列表删除，并删除定时器。 */ ipq_kill(qp); ecn = ip_frag_ecn_table[qp->ecn]; if (unlikely(ecn == 0xff)) { err = -EINVAL; goto out_fail; } /* Make the one we just received the head. */ if (prev) { head = prev->next; fp = skb_clone(head, GFP_ATOMIC); if (!fp) goto out_nomem; fp->next = head->next; if (!fp->next) qp->q.fragments_tail = fp; prev->next = fp; skb_morph(head, qp->q.fragments); head->next = qp->q.fragments->next; consume_skb(qp->q.fragments); qp->q.fragments = head; } WARN_ON(!head); WARN_ON(FRAG_CB(head)->offset != 0); /* Allocate a new buffer for the datagram. 计算原始报文的长度 超过64 KB*/ ihlen = ip_hdrlen(head); len = ihlen + qp->q.len; err = -E2BIG; if (len > 65535) goto out_oversize; /* Head of list must not be cloned. * 在组装分片时，所有的分片都会组装到第一个分片 * 上，因此第一个分片是不能克隆的，如果是克隆的， * 则需为分片组装重新分配一个SKB。 */ if (skb_unclone(head, GFP_ATOMIC)) goto out_nomem; /* If the first fragment is fragmented itself, we split * it to two chunks: the first with data and paged part * and the second, holding only fragments. */ /* * 分片队列的第一个SKB不能既带有数据，又带有分片，即其 * frag_list上不能有分片skb，如果有则重新分配一个SKB。最终的 * 效果是，head自身不包括数据，其frag_list上链接着所有分片的 * SKB。这也是SKB的一种表现形式，不一定是一个连续的数据块， * 但最终会调用skb_linearize()将这些数据都复制到一个连续的数据 * 块中。 */ if (skb_has_frag_list(head)) { struct sk_buff *clone; int i, plen = 0; clone = alloc_skb(0, GFP_ATOMIC); if (!clone) goto out_nomem; clone->next = head->next; head->next = clone; skb_shinfo(clone)->frag_list = skb_shinfo(head)->frag_list; skb_frag_list_init(head); for (i = 0; i < skb_shinfo(head)->nr_frags; i++) plen += skb_frag_size(&skb_shinfo(head)->frags[i]); clone->len = clone->data_len = head->data_len - plen; head->data_len -= clone->len; head->len -= clone->len; clone->csum = 0; clone->ip_summed = head->ip_summed; add_frag_mem_limit(qp->q.net, clone->truesize); } /* * 把所有分片组装起来即将分片链接到第一个 * SKB的frag_list上，同时还需要遍历所有分片， * 重新计算IP数据包长度以及校验和等。 */ skb_shinfo(head)->frag_list = head->next; skb_push(head, head->data - skb_network_header(head)); for (fp=head->next; fp; fp = fp->next) { head->data_len += fp->len; head->len += fp->len; if (head->ip_summed != fp->ip_summed) head->ip_summed = CHECKSUM_NONE; else if (head->ip_summed == CHECKSUM_COMPLETE) head->csum = csum_add(head->csum, fp->csum); head->truesize += fp->truesize; } /* * 重置首部长度、片偏移、标志位和总长度。 */ sub_frag_mem_limit(qp->q.net, head->truesize); head->next = NULL; head->dev = dev; head->tstamp = qp->q.stamp; IPCB(head)->frag_max_size = max(qp->max_df_size, qp->q.max_size); iph = ip_hdr(head); iph->tot_len = htons(len); iph->tos |= ecn; /* When we set IP_DF on a refragmented skb we must also force a * call to ip_fragment to avoid forwarding a DF-skb of size s while * original sender only sent fragments of size f (where f < s). * * We only set DF/IPSKB_FRAG_PMTU if such DF fragment was the largest * frag seen to avoid sending tiny DF-fragments in case skb was built * from one very small df-fragment and one large non-df frag. */ if (qp->max_df_size == qp->q.max_size) { IPCB(head)->flags |= IPSKB_FRAG_PMTU; iph->frag_off = htons(IP_DF); } else { iph->frag_off = 0; } ip_send_check(iph); __IP_INC_STATS(net, IPSTATS_MIB_REASMOKS); /* * 既然各分片都已处理完，释放ipq的分片队列。 */ qp->q.fragments = NULL; qp->q.fragments_tail = NULL; return 0; out_nomem: net_dbg_ratelimited("queue_glue: no memory for gluing queue %p\n", qp); err = -ENOMEM; goto out_fail; out_oversize: net_info_ratelimited("Oversized IP packet from %pI4\n", &qp->saddr); out_fail: __IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS); return err; }
1/4/4 ipq散列表重组
static void inet_frag_secret_rebuild(struct inet_frags *f) { int i; write_seqlock_bh(&f->rnd_seqlock);//顺序锁 if (!inet_frag_may_rebuild(f)) goto out; /* 获取新的用于计算hash的随机值 */ get_random_bytes(&f->rnd, sizeof(u32)); for (i = 0; i < INETFRAGS_HASHSZ; i++) { struct inet_frag_bucket *hb; struct inet_frag_queue *q; struct hlist_node *n; hb = &f->hash[i]; /* 取的桶节点 */ spin_lock(&hb->chain_lock); hlist_for_each_entry_safe(q, n, &hb->chain, list) { unsigned int hval = inet_frag_hashfn(f, q); if (hval != i) {/* 节点不属于当前桶 */ struct inet_frag_bucket *hb_dest; hlist_del(&q->list); /* 从当前桶中删除该节点 */ /* Relink to new hash chain. */ hb_dest = &f->hash[hval]; /* 找到目标桶 */ /* This is the only place where we take * another chain_lock while already holding * one. As this will not run concurrently, * we cannot deadlock on hb_dest lock below, if its * already locked it will be released soon since * other caller cannot be waiting for hb lock * that we've taken above. */ spin_lock_nested(&hb_dest->chain_lock, SINGLE_DEPTH_NESTING);/* 节点加入目标桶的链表中 */ hlist_add_head(&q->list, &hb_dest->chain); spin_unlock(&hb_dest->chain_lock); } } spin_unlock(&hb->chain_lock); } /* 设置重建标记和重建时间 */ f->rebuild = false; f->last_rebuild_jiffies = jiffies; out: write_sequnlock_bh(&f->rnd_seqlock); }
1/4/5 超时IP分片的清除
会定时清除规定 时间内没有完成重组的upq及其所有的分片
/* * Oops, a fragment queue timed out. Kill it and send an ICMP reply. */ static void ip_expire(unsigned long arg) { struct ipq *qp; struct net *net; qp = container_of((struct inet_frag_queue *) arg, struct ipq, q); net = container_of(qp->q.net, struct net, ipv4.frags); spin_lock(&qp->q.lock); //ipq 已经是complete状态不处理 直接释放ipq以及其所有的分片 if (qp->q.flags & INET_FRAG_COMPLETE) goto out; ipq_kill(qp);//将其从散列表移除 __IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS);//数据统计 if (!inet_frag_evicting(&qp->q)) {//在回收队列中 struct sk_buff *head = qp->q.fragments; const struct iphdr *iph; int err; __IP_INC_STATS(net, IPSTATS_MIB_REASMTIMEOUT); if (!(qp->q.flags & INET_FRAG_FIRST_IN) || !qp->q.fragments) goto out; rcu_read_lock(); head->dev = dev_get_by_index_rcu(net, qp->iif); if (!head->dev) goto out_rcu_unlock; /* skb has no dst, perform route lookup again */ iph = ip_hdr(head); err = ip_route_input_noref(head, iph->daddr, iph->saddr, iph->tos, head->dev); if (err) goto out_rcu_unlock; /* Only an end host needs to send an ICMP * "Fragment Reassembly Timeout" message, per RFC792. */ if (frag_expire_skip_icmp(qp->user) && (skb_rtable(head)->rt_type != RTN_LOCAL)) goto out_rcu_unlock; /* Send an ICMP "Fragment Reassembly Timeout" message. 发送ICMP 报文*/ icmp_send(head, ICMP_TIME_EXCEEDED, ICMP_EXC_FRAGTIME, 0); out_rcu_unlock: rcu_read_unlock(); } out: spin_unlock(&qp->q.lock); ipq_put(qp); }
1/4/6 进行节点回收工作队列
为了控制ip组装所占用的内存，设置了两个阈值low_thresh 、high_thresh 当前ipq散列表所占用的内存存储在 mem变量中，这些全局变量存在如下结构中（netns_frags）
struct netns_frags { /* The percpu_counter "mem" need to be cacheline aligned. * mem.count must not share cacheline with other writers */ struct percpu_counter mem ____cacheline_aligned_in_smp; /* sysctls */ int timeout; int high_thresh; int low_thresh; int max_dist; };
当mem大于high_thres 时，需要对散列表清理，直到mem值降低到low_thres。这两个值可以通过proc修改
static unsigned int inet_evict_bucket(struct inet_frags *f, struct inet_frag_bucket *hb) { struct inet_frag_queue *fq; struct hlist_node *n; unsigned int evicted = 0; HLIST_HEAD(expired); spin_lock(&hb->chain_lock); /* 遍历桶下的链表 */ hlist_for_each_entry_safe(fq, n, &hb->chain, list) { if (!inet_fragq_should_evict(fq))/* 未超过限定，无需回收 */ continue; if (!del_timer(&fq->timer)) /* 定时器无法删除 */ continue; /* 能够回收的节点加入到临时hash */ hlist_add_head(&fq->list_evictor, &expired); ++evicted; } spin_unlock(&hb->chain_lock); /* 依次调用回收函数进行回收 */ hlist_for_each_entry_safe(fq, n, &expired, list_evictor) f->frag_expire((unsigned long) fq); return evicted; } static void inet_frag_worker(struct work_struct *work) { /* 本次回收的桶节点数 */ unsigned int budget = INETFRAGS_EVICT_BUCKETS; unsigned int i, evicted = 0; struct inet_frags *f; f = container_of(work, struct inet_frags, frags_work); BUILD_BUG_ON(INETFRAGS_EVICT_BUCKETS >= INETFRAGS_HASHSZ); local_bh_disable(); /* 从上次回收完的下一个节点开始，进行回收 */ for (i = ACCESS_ONCE(f->next_bucket); budget; --budget) { evicted += inet_evict_bucket(f, &f->hash[i]); /* 回收并统计回收数量 */ i = (i + 1) & (INETFRAGS_HASHSZ - 1); /* 回收节点数超过最大值，停止 */ if (evicted > INETFRAGS_EVICT_MAX) break; } f->next_bucket = i; /* 记录下次需要开始回收的桶节点 */ local_bh_enable(); /* 如果需要重建，则重建 */ if (f->rebuild && inet_frag_may_rebuild(f)) inet_frag_secret_rebuild(f); }