1. tcp协议内部绑定tcp_hashinfo
struct proto tcp_prot = {
.name = "TCP",
.owner = THIS_MODULE,
.close = tcp_close,
.connect = tcp_v4_connect,
.disconnect = tcp_disconnect,
.accept = inet_csk_accept,
.ioctl = tcp_ioctl,
.init = tcp_v4_init_sock,
.destroy = tcp_v4_destroy_sock,
.shutdown = tcp_shutdown,
.setsockopt = tcp_setsockopt,
.getsockopt = tcp_getsockopt,
.recvmsg = tcp_recvmsg,
.sendmsg = tcp_sendmsg,
.sendpage = tcp_sendpage,
.backlog_rcv = tcp_v4_do_rcv,
.release_cb = tcp_release_cb,
.mtu_reduced = tcp_v4_mtu_reduced,
.hash = inet_hash,
.unhash = inet_unhash,
.get_port = inet_csk_get_port,
.enter_memory_pressure = tcp_enter_memory_pressure,
.sockets_allocated = &tcp_sockets_allocated,
.orphan_count = &tcp_orphan_count,
.memory_allocated = &tcp_memory_allocated,
.memory_pressure = &tcp_memory_pressure,
.sysctl_wmem = sysctl_tcp_wmem,
.sysctl_rmem = sysctl_tcp_rmem,
.max_header = MAX_TCP_HEADER,
.obj_size = sizeof(struct tcp_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.twsk_prot = &tcp_timewait_sock_ops,
.rsk_prot = &tcp_request_sock_ops,
.h.hashinfo = &tcp_hashinfo,
.no_autobind = true,
#ifdef CONFIG_COMPAT
.compat_setsockopt = compat_tcp_setsockopt,
.compat_getsockopt = compat_tcp_getsockopt,
#endif
#ifdef CONFIG_MEMCG_KMEM
.init_cgroup = tcp_init_cgroup,
.destroy_cgroup = tcp_destroy_cgroup,
.proto_cgroup = tcp_proto_cgroup,
#endif
};
EXPORT_SYMBOL(tcp_prot);其中有如下成员初始化
.h.hashinfo = &tcp_hashinfo,而tcp_hashinfo的结构体类型如下
struct inet_hashinfo tcp_hashinfo;
EXPORT_SYMBOL(tcp_hashinfo);2. struct inet_hashinfo 结构体定义
struct inet_hashinfo {
/* This is for sockets with full identity only. Sockets here will
* always be without wildcards and will have the following invariant:
*
* TCP_ESTABLISHED <= sk->sk_state < TCP_CLOSE
*
* TIME_WAIT sockets use a separate chain (twchain).
*/
struct inet_ehash_bucket *ehash; //established hash链表
spinlock_t *ehash_locks;
unsigned int ehash_mask; //内核打印信息为512
unsigned int ehash_locks_mask;
/* Ok, let's try this, I give up, we do need a local binding
* TCP hash as well as the others for fast bind/connect.
*/
struct inet_bind_hashbucket *bhash; //绑定的bind hash桶链表
unsigned int bhash_size; //上面 bhash 指针的个数，即所能bind hashbucket，内核打印为512
/* 4 bytes hole on 64 bit */
struct kmem_cache *bind_bucket_cachep;
/* All the above members are written once at bootup and
* never written again _or_ are predominantly read-access.
*
* Now align to a new cache line as all the following members
* might be often dirty.
*/
/* All sockets in TCP_LISTEN state will be in here. This is the only
* table where wildcard'd TCP sockets can exist. Hash function here
* is just local port number.
*/
struct inet_listen_hashbucket listening_hash[INET_LHTABLE_SIZE]
____cacheline_aligned_in_smp;
atomic_t bsockets;
};在这个结构体内部主要定义了如下成员
a. established hash桶及相关参数
struct inet_ehash_bucket *ehash; //established hash桶
spinlock_t *ehash_locks; //established 锁
unsigned int ehash_mask; //established屏蔽字，内核打印信息为512
unsigned int ehash_locks_mask; //established屏蔽字锁其中inet_ehash_bucket桶定义如下
struct inet_ehash_bucket {
struct hlist_nulls_head chain;
struct hlist_nulls_head twchain;
};struct hlist_nulls_head {
struct hlist_nulls_node *first; //节点
};struct hlist_nulls_node {
struct hlist_nulls_node *next, **pprev; //hash链表指针成员
};b. bind hash桶及相应的内存缓存指针
struct inet_bind_hashbucket *bhash; //bind hash桶链表
unsigned int bhash_size; //上面 bind bhash 指针的个数，内核打印为512
/* 4 bytes hole on 64 bit */
struct kmem_cache *bind_bucket_cachep; //bind桶内存缓存其中inet_bind_hashbucket bind hash桶定义如下
struct inet_bind_hashbucket {
spinlock_t lock;
struct hlist_head chain;
};
struct hlist_head {
struct hlist_node *first;
};
struct hlist_node {
struct hlist_node *next, **pprev;
};3. struct inet_hashinfo 结构体成员初始化
源码路径： linux-3.10.x\net\ipv4\tcp.c
源码调用： inet_init()-->tcp_init()
void __init tcp_init(void)
{
struct sk_buff *skb = NULL;
unsigned long limit;
int max_rshare, max_wshare, cnt;
unsigned int i;
BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb));
percpu_counter_init(&tcp_sockets_allocated, 0); //TCP socket的个数
percpu_counter_init(&tcp_orphan_count, 0); //tcp 孤儿的个数
//分配一个 tcp_hashinfo.bind_bucket_cachep 内存缓存
tcp_hashinfo.bind_bucket_cachep =
kmem_cache_create("tcp_bind_bucket",
sizeof(struct inet_bind_bucket), 0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
/* Size and allocate the main established and bind bucket
* hash tables.
*
* The methodology is similar to that of the buffer cache.
*/
//分配一个 established bucket链表
tcp_hashinfo.ehash =
alloc_large_system_hash("TCP established",
sizeof(struct inet_ehash_bucket),
thash_entries,
17, /* one slot per 128 KB of memory */
0,
NULL,
//返回ehash_mask屏蔽字
&tcp_hashinfo.ehash_mask,
0,
thash_entries ? 0 : 512 * 1024);
//遍历ehash_mask屏蔽字
for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) {
//初始化链表
INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].twchain, i);
}
//分配并初始化锁，及锁的个数
if (inet_ehash_locks_alloc(&tcp_hashinfo)) //ehash锁分配
panic("TCP: failed to alloc ehash_locks");
//---------------------------------------------------------------------
//分配bind hash桶链表
tcp_hashinfo.bhash =
alloc_large_system_hash("TCP bind",
sizeof(struct inet_bind_hashbucket),
tcp_hashinfo.ehash_mask + 1, //512+1
17, /* one slot per 128 KB of memory */
0,
//内核日志打印 tcp_hashinfo.bhash_size 为512
&tcp_hashinfo.bhash_size,
NULL,
0,
64 * 1024);
tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
for (i = 0; i < tcp_hashinfo.bhash_size; i++) { //遍历绑定的hashbucket
spin_lock_init(&tcp_hashinfo.bhash[i].lock); //初始化 bind hash链表的锁
INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); //初始化 bind hash链表
}
cnt = tcp_hashinfo.ehash_mask + 1;
tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
sysctl_tcp_max_orphans = cnt / 2;
sysctl_max_syn_backlog = max(128, cnt / 256);
tcp_init_mem(&init_net);
/* Set per-socket limits to no more than 1/128 the pressure threshold */
limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
max_wshare = min(4UL*1024*1024, limit);
max_rshare = min(6UL*1024*1024, limit);
sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
sysctl_tcp_wmem[1] = 16*1024;
sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
sysctl_tcp_rmem[1] = 87380;
sysctl_tcp_rmem[2] = max(87380, max_rshare);
pr_info("Hash tables configured (established %u bind %u)\n",
tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
//内核输出如上信息：“TCP: Hash tables configured (established 512 bind 512)”
tcp_metrics_init();
tcp_register_congestion_control(&tcp_reno);
tcp_tasklet_init();
}
4. inet_csk_get_port()
4.1 源码分析
前面在1~3所描述的内容是为4做“嫁衣”，现在开始inet_csk_get_port()源码分析
int inet_csk_get_port(struct sock *sk, unsigned short snum)
{
//得到该sock对应协议族的全局的底层容器 hashinfo = tcp_hashinfo ，
//其中它在struct proto tcp_prot内部初始化。而tcp_hashinfo的部分成
//员是在 tcp_init()函数内部初始化，要搞清楚这里的关系，一定要查看
//tcp_init() 函数内部的实现
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
struct inet_bind_hashbucket *head;
struct inet_bind_bucket *tb;
int ret, attempts = 5;
struct net *net = sock_net(sk);
int smallest_size = -1, smallest_rover;
kuid_t uid = sock_i_uid(sk); //获取当前socket对应的用户id
local_bh_disable();
//端口无效（我们的应用程序在开发的时候配置的无效端口，所以这里会随机
//分配一个），这种情况就是随机绑定一个没有使用的端口
if (!snum) { //端口无效
int remaining, rover, low, high;
again:
inet_get_local_port_range(&low, &high); //获取端口范围，一般就是1到65535，就是我们常用的端口号范围，当然也可以自己配置
remaining = (high - low) + 1; //剩余端口个数
smallest_rover = rover = net_random() % remaining + low; //随机分配一个数字作为端口
smallest_size = -1;
do {
//是否是保留的端口
if (inet_is_reserved_local_port(rover))
goto next_nolock; //如果是保留的端口就切换到下一个，即++rover
//通过端口号，即哈希值，确定其所在的链表head
head = &hashinfo->bhash[inet_bhashfn(net, rover,
hashinfo->bhash_size)];
/* 锁住哈希桶 */
spin_lock(&head->lock);
/* 从头遍历哈希桶，在inet_bind_bucket_for_each函数内部运用了
container_of机制，通过指针成员获取其对应的结构体，这里既是tb*/
inet_bind_bucket_for_each(tb, &head->chain)
/* 如果端口被使用了,就进行冲突检测 */
if (net_eq(ib_net(tb), net) && tb->port == rover) {
if (((tb->fastreuse > 0 && //tb中的参数可“快速重用”
sk->sk_reuse && //socket参数允许快速重用
sk->sk_state != TCP_LISTEN) || //不在监听状态
(tb->fastreuseport > 0 &&
sk->sk_reuseport &&
uid_eq(tb->fastuid, uid))) && //socket用户id相等
(tb->num_owners < smallest_size || smallest_size == -1)) {
smallest_size = tb->num_owners; /* 记下这个端口使用者的个数 */
smallest_rover = rover; /* 记下这个端口 */
/* 如果系统绑定的端口已经很多了，那么就判断端口是否有绑定冲突*/
if (atomic_read(&hashinfo->bsockets) > (high - low) + 1 &&
!inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, false)) {
snum = smallest_rover; /* 没有冲突，使用此端口 */
goto tb_found;
}
}
/* 检查是否有端口绑定冲突，该端口是否能重用 */
if (!inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, false)) {
snum = rover;
goto tb_found;
}
goto next; /* 此端口不可重用，看下一个 */
}
/* 找到了没被用的端口，退出 */
break; //如果一个桶遍历过了，没有冲突的，那么就需要在下面建立一个inet_bind_bucket
next:
spin_unlock(&head->lock);
next_nolock:
if (++rover > high)
rover = low;
} while (--remaining > 0);
/* Exhausted local port range during search? It is not
* possible for us to be holding one of the bind hash
* locks if this test triggers, because if 'remaining'
* drops to zero, we broke out of the do/while loop at
* the top level, not from the 'break;' statement.
*/
ret = 1;
if (remaining <= 0) {
if (smallest_size != -1) {
snum = smallest_rover;
goto have_snum;
}
goto fail;
}
/* OK, here is the one we will use. HEAD is
* non-NULL and we hold it's mutex.
*/
snum = rover; /* 自动选择的可用端口 */
} else { /* 如果应用层有指定要绑定的端口 */
have_snum: //有端口
/* 走到这里，表示用户已经自己绑定了端口
1. inet_bhashfn(net, snum, hashinfo->bhash_size): 计算struct inet_bind_hashbucket指针索引
2. head = &hashinfo->bhash[*]: 返回struct inet_bind_hashbucket hash桶指针，即端口所在的哈希桶
3. inet_bind_bucket_for_each(tb, &head->chain)：遍历当前hash桶内部的chain（hlist）链表，该链表
上注册了已被绑定端口，通过该chain链表及node成员找到（运用container_of）找到所属的结构体，即
结构体为tb （struct inet_bind_bucket），具体的端口绑定到链表详见inet_bind_bucket_create()函
数内部的实现
4. net_eq(ib_net(tb), net) && tb->port == snum： 是否是同一个net[个人理解，这个应该是创建一个socket就对应一个net] && 端口是否相等
*/
head = &hashinfo->bhash[inet_bhashfn(net, snum,
hashinfo->bhash_size)];
spin_lock(&head->lock);
inet_bind_bucket_for_each(tb, &head->chain)
if (net_eq(ib_net(tb), net) && tb->port == snum) //从hash链表里获取的端口与应用配置的端口相等？
goto tb_found; /* 发现端口在用 */
}
tb = NULL;
goto tb_not_found;
tb_found:
/* 端口上有绑定sock时 */
if (!hlist_empty(&tb->owners)) { //为NULL表示tb未被使用
/* 这是强制的绑定啊，不管端口是否会绑定冲突！*/
if (sk->sk_reuse == SK_FORCE_REUSE)
goto success;
//根据socket的参数判断当前的端口是否快速重用
if (((tb->fastreuse > 0 &&
sk->sk_reuse && sk->sk_state != TCP_LISTEN) ||
(tb->fastreuseport > 0 &&
sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
smallest_size == -1) { /* 指定端口的情况 */
goto success;
} else {
ret = 1;
if (inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, true)) { /* 端口绑定冲突 */
/* 自动分配的端口绑定冲突了，再次尝试，最多重试5次。
* 我觉得以下if不必要，因为自动选择时goto tb_found之前都有检测过了。
*/
if (((sk->sk_reuse && sk->sk_state != TCP_LISTEN) ||
(tb->fastreuseport > 0 &&
sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
smallest_size != -1 && --attempts >= 0) {
spin_unlock(&head->lock);
goto again;
}
goto fail_unlock;
}
}
}
tb_not_found: //到这里表示在hash桶里面没有找到端口
ret = 1;
/* 申请和初始化一个inet_bind_bucket结构， 返回一个tb hash桶*/
if (!tb && (tb = inet_bind_bucket_create(hashinfo->bind_bucket_cachep,
net, head, snum)) == NULL)
goto fail_unlock;
if (hlist_empty(&tb->owners)) { //在inet_bind_bucket_create()函数内部tb->owners初始化为NULL
if (sk->sk_reuse && sk->sk_state != TCP_LISTEN) //sk->sk_reuse变量在inet_create()函数内部初始化的
tb->fastreuse = 1;
else
tb->fastreuse = 0;
if (sk->sk_reuseport) { //端口重用
tb->fastreuseport = 1;
tb->fastuid = uid; //用户id
} else
tb->fastreuseport = 0;
} else {
if (tb->fastreuse && //重用
(!sk->sk_reuse || sk->sk_state == TCP_LISTEN)) //禁止端口复用 || socket状态为监听
tb->fastreuse = 0; //禁止重用
if (tb->fastreuseport &&
(!sk->sk_reuseport || !uid_eq(tb->fastuid, uid))) //端口禁止重用 || 用户id不相等
tb->fastreuseport = 0;
}
success:
/* 赋值icsk中的inet_bind_bucket */
if (!inet_csk(sk)->icsk_bind_hash) //未绑定hash桶, 在下面的 inet_bind_hash()函数内部绑定
inet_bind_hash(sk, tb, snum); //重要，将hash桶绑定到sk->sk_prot->h.hashinfo上
WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
ret = 0;
fail_unlock:
spin_unlock(&head->lock);
fail:
local_bh_enable();
return ret;
}
EXPORT_SYMBOL_GPL(inet_csk_get_port);该函数的主要功能是通过协议层bind()函数绑定的本地端口判断，端口是否为无效？
a. 端口无效就使用inet_get_local_port_range()自动分配一个端口，否则就使用应用层经过系统调用传入的端口进行绑定
b. 端口已经分配或协议层传入的有效就进行端口有效性判断，确定端口是否在使用并且申请该端口的用户id使用相同，如果端口未被使用就进入到d. 否则进入c.
head = &hashinfo->bhash[inet_bhashfn(net, snum,
hashinfo->bhash_size)];
spin_lock(&head->lock);
inet_bind_bucket_for_each(tb, &head->chain)
if (net_eq(ib_net(tb), net) && tb->port == snum) //从hash链表里获取的端口与应用配置的端口相等？
goto tb_found; /* 发现端口在用 */ c. 判定该端口对应的tb（struct inet_bind_bucket）是否在使用、确定该端口是否可重用、在进行冲突检测
if (!hlist_empty(&tb->owners)) { //为NULL表示tb未被使用
/* 这是强制的绑定啊，不管端口是否会绑定冲突！*/
if (sk->sk_reuse == SK_FORCE_REUSE)
goto success;d. 当端口未被使用就申请一个inet_bind_bucket bind桶，保存当前应用层的端口，最后将端口信息添加到hash桶链表上，详见下面的流程图结构
/* 申请和初始化一个inet_bind_bucket结构， 返回一个tb hash桶*/
if (!tb && (tb = inet_bind_bucket_create(hashinfo->bind_bucket_cachep,
net, head, snum)) == NULL)struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
struct net *net,
struct inet_bind_hashbucket *head,
const unsigned short snum)
{
struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
if (tb != NULL) {
write_pnet(&tb->ib_net, hold_net(net)); //tb->ib_net指向命名空间为net
tb->port = snum; //绑定端口
tb->fastreuse = 0;
tb->fastreuseport = 0;
tb->num_owners = 0;
INIT_HLIST_HEAD(&tb->owners); //初始化tb->owners链表为NULL
hlist_add_head(&tb->node, &head->chain); //将分配的节点tb->node添加到hash桶链表head->chain上
}
return tb;
}e. 当申请一个inet_bind_bucket tb桶成功后，通过inet_bind_hash(sk, tb, snum)将该tb添加到sk->sk_bind_node链表上
f. 至此，就完成了端口在hash桶及sock上的绑定。
4.2 sock与inet_bind_hashbucket与inet_bind_bucket的关系图
这里有个核心的功能，就是通过结构体的成员获取该结构体的指针
inet_bind_bucket_for_each(tb, &head->chain)struct inet_bind_hashbucket *head;
struct inet_bind_bucket *tb;struct inet_bind_hashbucket {
spinlock_t lock;
struct hlist_head chain;
};
struct inet_bind_bucket {
#ifdef CONFIG_NET_NS
struct net *ib_net; //网络命名空间net
#endif
unsigned short port; //端口
signed char fastreuse; //快速重用，初始化为0
signed char fastreuseport; //快速重用端口，初始化0
kuid_t fastuid;
int num_owners; //初始化为0
struct hlist_node node;
struct hlist_head owners;
};
inet_bind_bucket_for_each(tb, &head->chain)#define inet_bind_bucket_for_each(tb, head) \
hlist_for_each_entry(tb, head, node)#define hlist_for_each_entry(pos, head, member) \
for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\
pos; \
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))#define hlist_entry_safe(ptr, type, member) \
({ typeof(ptr) ____ptr = (ptr); \
____ptr ? hlist_entry(____ptr, type, member) : NULL; \
})#define hlist_entry(ptr, type, member) container_of(ptr,type,member)另外这位大神写的很详细：点击打开链接