TCP服务器 IOCP设计 讨论

高性能的socket 服务器设计比较复杂，有的理解起来很困难。目前，在window下socket服务器，普遍评价比较好的使用使用IO完成端口（IOCP）和socket结合起来，效率和速度都可以兼顾。
对这种设计，对于动态内存的分配和释放，理解起来比较困难。
c++ 网络编程（九）TCP/IP windows下 多线程超详细教程 以及 多线程实现服务端
https://blog.csdn.net/qq_42564846/article/details/82736100
这篇文章对帮助IOCP的理解很好，可能作者对内存操作忽略了一些东西。

下面的代码进行了修改。
目的：
用于socket 编程学习，使用了IO完成端口的 TCP服务。
该程序为一个聊天室服务程序，和客户端配合使用。

代码适用范围：
win32 下的编程， 可以使用VS2010 ，VS2015直接编译（生成空白project后加入该文件即可）。

修改内容：
1.和原文相比修改了原文的一些bug，特别是内存分配的问题（原链接中的代码有内存不停增长的问题）。
2.添加了一些输出，可以观察线程和内存的关系。

服务器端代码：
#define _CRT_SECURE_NO_WARNINGS
#define _WINSOCK_DEPRECATED_NO_WARNINGS

#include <stdio.h>
#include <stdlib.h>
#include <process.h>
#include <winsock2.h>
#include <windows.h>
#include
#include <conio.h>

/*c++ 网络编程（九）TCP/IP LINUX/windows下 多线程超详细教程 以及 多线程实现服务端
https://blog.csdn.net/qq_42564846/article/details/82736100
*/
#pragma comment(lib,“ws2_32.lib”); //加载ws2_32.dll

#define BUF_SIZE 100
#define READ 3
#define WRITE 5

typedef struct // socket info
{
SOCKET hClntSock;
SOCKADDR_IN clntAdr;
} PER_HANDLE_DATA, *LPPER_HANDLE_DATA;

typedef struct // buffer info
{
OVERLAPPED overlapped;
WSABUF wsaBuf;
char buffer[BUF_SIZE];
int rwMode; // READ or WRITE 读写模式
} PER_IO_DATA, *LPPER_IO_DATA;

unsigned int WINAPI EchoThreadMain(LPVOID CompletionPortIO);
void ErrorHandling(char *message);
SOCKET ALLCLIENT[100];
int clientcount = 0;
HANDLE hMutex;//互斥量

int gExit=0; //退出程序

//退出消息获取程序
BOOL WINAPI CtrlExitHandler(DWORD fdwctrltype)
{
switch (fdwctrltype)
{
// handle the ctrl-c signal.
case CTRL_C_EVENT:
printf(“ctrl-c event\n\n”);
gExit =1;
return(true);
// ctrl-close: confirm that the user wants to exit.
case CTRL_CLOSE_EVENT:
printf(“ctrl-close event\n\n”);//点击了系统窗口的关闭
OutputDebugString(_T(“ctrl-close event\n”));
return(true);
// pass other signals to the next handler.
case CTRL_BREAK_EVENT:
printf(“ctrl-break event\n\n”);
return false;
case CTRL_LOGOFF_EVENT:
printf(“ctrl-logoff event\n\n”);
return false;
case CTRL_SHUTDOWN_EVENT:
printf(“ctrl-shutdown event\n\n”);
return false;
case WM_CLOSE:
printf(“wm close event\n\n”);
return true;
case WM_QUIT:
printf(“system shutdown event\n\n”);
return true;
default:
return false;
}
}

int main(int argc, char* argv[])
{
hMutex = CreateMutex(NULL, FALSE, NULL);//创建互斥量

WSADATA    wsaData;
HANDLE hComPort;
SYSTEM_INFO sysInfo;
LPPER_IO_DATA ioInfo;
LPPER_HANDLE_DATA handleInfo;

SOCKET hServSock;
SOCKADDR_IN servAdr;
int  i;
DWORD recvBytes = 0, flags = 0;

if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
	ErrorHandling("WSAStartup() error!");

hComPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);//创建CP对象
GetSystemInfo(&sysInfo);//获取当前系统的信息

for (i = 0; i < sysInfo.dwNumberOfProcessors; i++)
	_beginthreadex(NULL, 0, EchoThreadMain, (LPVOID)hComPort, 0, NULL);//创建=CPU个数的线程数

hServSock = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);//不是非阻塞套接字，但是重叠IO套接字。
memset(&servAdr, 0, sizeof(servAdr));
servAdr.sin_family = AF_INET;
servAdr.sin_addr.s_addr = htonl(INADDR_ANY);
servAdr.sin_port = htons(1234);

bind(hServSock, (SOCKADDR*)&servAdr, sizeof(servAdr));
listen(hServSock, 5);

if (SetConsoleCtrlHandler((PHANDLER_ROUTINE)CtrlExitHandler, true))
{
	printf("  the control handler is installed.\n");
}

gExit = 0;
while (gExit == 0)
{
	{//q/Q 按键 退出程序。按键后，当有新的TCP联机才会退出。
		while (_kbhit() != 0) {//按下q 或Q 后，退出程序。非阻塞方式获取字符
			char c = _getch();
			if (c == 'q' || c == 'Q') {
				gExit = 1;
				//break; //去除后将，读取所有输入的字符。
			}
		}
		if (gExit == 1)  break;
	}
	SOCKET hClntSock;
	SOCKADDR_IN clntAdr;
	int addrLen = sizeof(clntAdr);
	if(gExit ==1)	break;//退出

	hClntSock = accept(hServSock, (SOCKADDR*)&clntAdr, &addrLen);

	handleInfo = (LPPER_HANDLE_DATA)malloc(sizeof(PER_HANDLE_DATA));//和重叠IO一样
	handleInfo->hClntSock = hClntSock;//存储客户端套接字

	WaitForSingleObject(hMutex, INFINITE);//线程同步
	{//新加入链接，存入套接字队列
		bool bInserted = false;
		for (int i = 0; i < clientcount;i++)
		{
			if (ALLCLIENT[i] == 0) {
				bInserted = true;
				ALLCLIENT[i] = hClntSock;//存入套接字队列，空白位置
				break;
			}
		}
		if (bInserted == false) {
			ALLCLIENT[clientcount++] = hClntSock;//存入套接字队列,最后位置
		}
	}
	ReleaseMutex(hMutex);

	memcpy(&(handleInfo->clntAdr), &clntAdr, addrLen);

	CreateIoCompletionPort((HANDLE)hClntSock, hComPort, (DWORD)handleInfo, 0);//连接套接字和CP对象
																			  //已完成信息将写入CP对象
	ioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));//存储接收到的信息
	memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
	ioInfo->wsaBuf.len = BUF_SIZE;
	ioInfo->wsaBuf.buf = ioInfo->buffer;//和重叠IO一样
	ioInfo->rwMode = READ;//读写模式

	WSARecv(handleInfo->hClntSock, &(ioInfo->wsaBuf),//非阻塞模式
		1, &recvBytes, &flags, &(ioInfo->overlapped), NULL);

	std::cout <<"thread:"<<(long)::GetCurrentThreadId()<<" new IO:"<<(long)ioInfo;//观察内存分配 
}
CloseHandle(hMutex);//销毁互斥量
{//cleanup network, 似乎不做这些操作也没问题！
	int iResult = closesocket(hServSock);
	if (iResult == SOCKET_ERROR) {
		wprintf(L"closesocket function failed with error %d\n", WSAGetLastError());
		WSACleanup();
		return 1;
	}
	WSACleanup();
	printf("press any key to close window.");
	_getch();
}
return 0;

}

unsigned int WINAPI EchoThreadMain(LPVOID pComPort)//线程的执行
{
HANDLE hComPort = (HANDLE)pComPort;
SOCKET sock;
DWORD bytesTrans;
LPPER_HANDLE_DATA handleInfo;
LPPER_IO_DATA ioInfo;
DWORD flags = 0;

char tmpstr[512];

while (1)//工作线程内部大循环
{
	GetQueuedCompletionStatus(hComPort, &bytesTrans,//确认“已完成”的I/O！！
		(LPDWORD)&handleInfo, (LPOVERLAPPED*)&ioInfo, INFINITE);//INFINITE使用时，程序将阻塞，直到已完成的I/O信息写入CP对象
	sock = handleInfo->hClntSock;//客户端套接字

	if (ioInfo->rwMode == READ)//读写模式（此时缓冲区有数据）
	{
		std::cout <<"message received! \n";//puts("message received!");
		if (bytesTrans == 0)    // 连接结束
		{
			WaitForSingleObject(hMutex, INFINITE);//线程同步

			closesocket(sock);
			for (int i = 0; i < clientcount; i++)
				if(ALLCLIENT[i] == sock)
					ALLCLIENT[i] = 0;//断开置0

			ReleaseMutex(hMutex);

			free(handleInfo); 
			free(ioInfo);
			std::cout <<"thread:"<<(long)::GetCurrentThreadId()<<" free IO:"<<(long)ioInfo << " close "<< (long)(handleInfo->clntAdr.sin_port);//观察内存释放 main
			continue;
		}
		int i = 0;

		for (; i < clientcount; i++)
		{
			if (ALLCLIENT[i] != 0)//判断是否为已连接的套接字
			{
				//if (ALLCLIENT[i] != sock)
				{
					//std::cout << handleInfo->clntAdr.sin_port << " send to other " << bytesTrans << "\n";
					LPPER_IO_DATA newioInfo;
					newioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));//动态分配内存
					memset(&(newioInfo->overlapped), 0, sizeof(OVERLAPPED));
					strncpy(newioInfo->buffer, ioInfo->buffer, bytesTrans);//重新构建新的内存，防止多次释放free
					newioInfo->wsaBuf.buf = newioInfo->buffer;
					newioInfo->wsaBuf.len = bytesTrans;
					newioInfo->rwMode = WRITE;

					strncpy(tmpstr,newioInfo->buffer,bytesTrans);
					tmpstr[bytesTrans]=0;

					WSASend(ALLCLIENT[i], &(newioInfo->wsaBuf),//回声
						1, NULL, 0, &(newioInfo->overlapped), NULL);
					//观察内存分配 ,收到后转发的信息。
					std::cout <<"send to "<<handleInfo->clntAdr.sin_port <<" len:"<<bytesTrans <<",thread:"<<(long)::GetCurrentThreadId()<<" new:"<<(long)newioInfo <<" "<<tmpstr;
				}
			}
		}
		
		//ioInfo = (LPPER_IO_DATA)malloc(sizeof(PER_IO_DATA));//动态分配内存
		//std::cout <<"thread:"<<(long)::GetCurrentThreadId()<<" new IO:"<<(long)ioInfo;//观察内存分配 
		memset(&(ioInfo->overlapped), 0, sizeof(OVERLAPPED));
		ioInfo->wsaBuf.len = BUF_SIZE;
		ioInfo->wsaBuf.buf = ioInfo->buffer;
		ioInfo->rwMode = READ;
		WSARecv(sock, &(ioInfo->wsaBuf),//继续非阻塞式接收
			1, NULL, &flags, &(ioInfo->overlapped), NULL);
	}
	else
	{
		//观察哪个thread释放了内存
		std::cout << handleInfo->clntAdr.sin_port <<"  thread:"<<(long)::GetCurrentThreadId() << " free:"<<(long)ioInfo  << "  message sent! \n";
		free(ioInfo);
	}
}
return 0;

}

void ErrorHandling(char *message)
{
fputs(message, stderr);
fputc(’\n’, stderr);
exit(1);
}

客户端代码，（基本上使用了原作者的代码）
#define _CRT_SECURE_NO_WARNINGS
#define _WINSOCK_DEPRECATED_NO_WARNINGS

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <windows.h>
#include <process.h>
#define BUF_SIZE 1000
#define NAME_SIZE 20

#pragma comment(lib, “ws2_32.lib”) //加载 ws2_32.dll

unsigned WINAPI SendMsg(void * arg);//发送信息函数
unsigned WINAPI RecvMsg(void * arg);//接受信息函数
void ErrorHandling(char * msg);//错误返回函数

int haveread = 0;
char NAME[50];//[名字]
char ANAME[50];
char msg[BUF_SIZE];//信息

int main(int argc, char *argv[])
{

printf("请输入网名：");
scanf("%s", NAME);
WSADATA wsaData;
SOCKET hSock;
SOCKADDR_IN servAdr;
HANDLE hSndThread, hRcvThread;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0)
	ErrorHandling("WSAStartup() error!");

hSock = socket(PF_INET, SOCK_STREAM, 0);
memset(&servAdr, 0, sizeof(servAdr));
servAdr.sin_family = AF_INET;
servAdr.sin_addr.s_addr = inet_addr("127.0.0.1");
servAdr.sin_port = htons(1234);

if (connect(hSock, (SOCKADDR*)&servAdr, sizeof(servAdr)) == SOCKET_ERROR)
	ErrorHandling("connect() error");

int resultsend;
puts("Welcome to joining our chatting room!\n");
sprintf(ANAME, "[%s]", NAME);

hSndThread =
	(HANDLE)_beginthreadex(NULL, 0, SendMsg, (void*)&hSock, 0, NULL);//写线程
hRcvThread =
	(HANDLE)_beginthreadex(NULL, 0, RecvMsg, (void*)&hSock, 0, NULL);//读线程

WaitForSingleObject(hSndThread, INFINITE);//等待线程结束
WaitForSingleObject(hRcvThread, INFINITE);
closesocket(hSock);
WSACleanup();
system("pause");
return 0;

}

unsigned WINAPI SendMsg(void * arg) // send thread main
{
SOCKET sock = ((SOCKET)arg);
char name_msg[NAME_SIZE + BUF_SIZE];
char padd[2];
fgets(padd, 2, stdin);//多余的’\n’
printf("\n send message:");
while (1)
{
{
fgets(msg, BUF_SIZE, stdin);
if (!strcmp(msg, “q\n”) || !strcmp(msg, “Q\n”))
{
closesocket(sock);
exit(0);
}
sprintf(name_msg, “[%s] %s”, NAME, msg);
char numofmsg = strlen(name_msg) + ‘0’;
char newmsg[100]; newmsg[0] = numofmsg; newmsg[1] = 0;//第一个字符表示消息的长度
strcat(newmsg, name_msg);
int result = send(sock, newmsg, strlen(newmsg), 0);
if (result == -1)return -1;//发送错误
}
}
return NULL;
}

unsigned WINAPI RecvMsg(void * arg) // read thread main
{
SOCKET sock = ((SOCKET)arg);
char name_msg[NAME_SIZE + BUF_SIZE];
int str_len = 0;
while (1)
{
{
char lyfstr[1000] = { 0 };
int totalnum = 0;
str_len = recv(sock, name_msg, 1, 0);//读取第一个字符！获取消息的长度
if (str_len == -1)//读取错误
{
printf(“return -1\n”);
return -1;
}
if (str_len == 0)//读取结束
{
printf(“return 0\n”);
return 0;//读取结束
}
totalnum = name_msg[0] - ‘0’;
int count = 0;

		do
		{
			str_len = recv(sock, name_msg, 1, 0);

			name_msg[str_len] = 0;

			if (str_len == -1)//读取错误
			{
				printf("return1 -1\n");
				return -1;
			}
			if (str_len == 0)
			{
				printf("return1 0\n");
				return 0;//读取结束
			}
			strcat(lyfstr, name_msg);
			count = str_len + count;

		} while (count < totalnum);

		lyfstr[count] = '\0';
		printf("\n");
		strcat(lyfstr, "\n");
		fputs(lyfstr, stdout);
		printf(" send message:");
		fflush(stdout);
		memset(name_msg, 0, sizeof(char));
	}
}
return NULL;

}

void ErrorHandling(char * msg)
{
fputs(msg, stderr);
fputc(’\n’, stderr);
exit(1);
}