TCP收消息
TCP收消息相关的类有TcpIOThread.TcpAcceptor.TcpRead
三个类。相互间的关系是:TcpIOThread
初始化TcpRead
和TcpAcceptor
,并将后两者关联起来。 这样Acceptor
接收到的连接直接放到TcpRead
中读取数据。
class TcpIOThread
{
public:
TcpIOThread(NetWork * poNetWork);
~TcpIOThread();
//初始化建立相关的关联关系
int Init(const std::string & sListenIp, const int iListenPort, const int iIOThreadCount);
void Start();
void Stop();
int AddMessage(const int iGroupIdx, const std::string & sIP, const int iPort, const std::string & sMessage);
private:
NetWork * m_poNetWork;
TcpAcceptor m_oTcpAcceptor; //连接建立类
std::vector<TcpRead *> m_vecTcpRead; //消息读取类
std::vector<TcpWrite *> m_vecTcpWrite; //消息发送
bool m_bIsStarted;
};
下面看一下他的初始化部分
int TcpIOThread :: Init(const std::string & sListenIp, const int iListenPort, const int iIOThreadCount)
{
//启动iIOThreadCount线程个读写线程
for (int i = 0; i < iIOThreadCount; i++)
{
TcpRead * poTcpRead = new TcpRead(m_poNetWork);
assert(poTcpRead != nullptr);
m_vecTcpRead.push_back(poTcpRead);
//将Read内部的EventLoop添加到Acceptor中,这样Acceptor接收消息之后可以直接将fd投放Read的EventLoop中直接使用
m_oTcpAcceptor.AddEventLoop(poTcpRead->GetEventLoop());
TcpWrite * poTcpWrite = new TcpWrite(m_poNetWork);
assert(poTcpWrite != nullptr);
m_vecTcpWrite.push_back(poTcpWrite);
}
//启动监听
m_oTcpAcceptor.Listen(sListenIp, iListenPort);
int ret = -1;
for (auto & poTcpRead : m_vecTcpRead)
{
//初始化Read
ret = poTcpRead->Init();
if (ret != 0)
{
return ret;
}
}
.........
return 0;
}
Acceptor
class TcpAcceptor : public Thread
{
public:
TcpAcceptor();
~TcpAcceptor();
//封装socket的listen函数
void Listen(const std::string & sListenIP, const int iListenPort);
//accept并将相应的fd,socketaddr创建Event添加到Event中
void run();
void Stop();
//将Read中的EventLoop添加到私有m_vecEventLoop中
void AddEventLoop(EventLoop * poEventLoop);
//添加到最空闲的读线程
void AddEvent(int iFD, SocketAddress oAddr);
private:
ServerSocket m_oSocket;
std::vector<EventLoop *> m_vecEventLoop;
private:
bool m_bIsEnd;
bool m_bIsStarted;
};
void TcpAcceptor :: Listen(const std::string & sListenIP, const int iListenPort)
{
m_oSocket.listen(SocketAddress(sListenIP, (unsigned short)iListenPort));
}
void TcpAcceptor :: run()
{
m_bIsStarted = true;
PLHead("start accept...");
m_oSocket.setAcceptTimeout(500);
m_oSocket.setNonBlocking(true);
while (true)
{
struct pollfd pfd;
int ret;
pfd.fd = m_oSocket.getSocketHandle();
pfd.events = POLLIN;
ret = poll(&pfd, 1, 500);
if (ret != 0 && ret != -1)
{
SocketAddress oAddr;
int fd = -1;
try
{
//accept函数封装
fd = m_oSocket.acceptfd(&oAddr);
}
catch(...)
{
fd = -1;
}
if (fd >= 0)
{
BP->GetNetworkBP()->TcpAcceptFd();
PLImp("accepted!, fd %d ip %s port %d",
fd, oAddr.getHost().c_str(), oAddr.getPort());
//将新建立的连接添加到Read的EventLoop中
AddEvent(fd, oAddr);
}
}
if (m_bIsEnd)
{
PLHead("TCP.Acceptor [END]");
return;
}
}
}
void TcpAcceptor :: AddEventLoop(EventLoop * poEventLoop)
{
m_vecEventLoop.push_back(poEventLoop);
}
void TcpAcceptor :: AddEvent(int iFD, SocketAddress oAddr)
{
EventLoop * poMinActiveEventLoop = nullptr;
int iMinActiveEventCount = 1 << 30;
//挑选最空闲的EventLoop也就是最空闲的Read进行添加。空闲指的是监听的事件数量最少
for (auto & poEventLoop : m_vecEventLoop)
{
int iActiveCount = poEventLoop->GetActiveEventCount();
if (iActiveCount < iMinActiveEventCount)
{
iMinActiveEventCount = iActiveCount;
poMinActiveEventLoop = poEventLoop;
}
}
//printf("this %p addevent %p fd %d ip %s port %d\n",
//this, poMinActiveEventLoop, iFD, oAddr.getHost().c_str(), oAddr.getPort());
poMinActiveEventLoop->AddEvent(iFD, oAddr);
}
OK,分析到这TCP接收的逻辑就全部完成了。Acceptor建立连接后将连接扔到监听事件最少Read的事件循环EventLoop中进行读事件的处理。若触发EPOLL则会接收信息并将信息通过NetWork的OnReceiveMessage将信息传递到Paxos的接收队列中。
TCP发消息
与Tcp发消息主要的类是TcpWrite
和TcpClient
。TcpClient是TcpWrite的私有成员,两者通过NetWork和EventLoop发生联系。其实TcpClient作用就是将发送消息和事件Event的添加封装起来。
class TcpClient
{
public:
TcpClient(
EventLoop * poEventLoop,
NetWork * poNetWork);
~TcpClient();
int AddMessage(const std::string & sIP, const int iPort, const std::string & sMessage);
void DealWithWrite();
private:
//从已经建立的Event中取出,不存在用下面的函数创建并返回
MessageEvent * GetEvent(const std::string & sIP, const int iPort);
MessageEvent * CreateEvent(const uint64_t llNodeID, const std::string & sIP, const int iPort);
private:
EventLoop * m_poEventLoop;
NetWork * m_poNetWork;
private:
std::map<uint64_t, MessageEvent *> m_mapEvent;
std::vector<MessageEvent *> m_vecEvent;
std::mutex m_oMutex;
};
TcpClient :: TcpClient(EventLoop * poEventLoop, NetWork * poNetWork)
: m_poEventLoop(poEventLoop), m_poNetWork(poNetWork)
{
m_vecEvent.reserve(1000);
}
TcpClient :: ~TcpClient()
{
//只有在每一次析构时才释放相关的资源,而且貌似没有对添加EPOLLOUT事件的移除逻辑(待查)
for (auto & it : m_mapEvent)
{
delete it.second;
}
}
int TcpClient :: AddMessage(const std::string & sIP, const int iPort, const std::string & sMessage)
{
//PLImp("ok");
MessageEvent * poEvent = GetEvent(sIP, iPort);
if (poEvent == nullptr)
{
PLErr("no event created for this ip %s port %d", sIP.c_str(), iPort);
return -1;
}
return poEvent->AddMessage(sMessage);
}
MessageEvent * TcpClient :: GetEvent(const std::string & sIP, const int iPort)
{
//根据IP和PORT生成NodeID,使用NodeID进行索引Event
uint32_t iIP = (uint32_t)inet_addr(sIP.c_str());
uint64_t llNodeID = (((uint64_t)iIP) << 32) | iPort;
std::lock_guard<std::mutex> oLockGuard(m_oMutex);
auto it = m_mapEvent.find(llNodeID);
if (it != end(m_mapEvent))
{
return it->second;
}
return CreateEvent(llNodeID, sIP, iPort);
}
MessageEvent * TcpClient :: CreateEvent(const uint64_t llNodeID, const std::string & sIP, const int iPort)
{
PLImp("start, ip %s port %d", sIP.c_str(), iPort);
//不存在新建连接并将将其添加到EventLoop中
Socket oSocket;
oSocket.setNonBlocking(true);
oSocket.setNoDelay(true);
SocketAddress oAddr(sIP, iPort);
oSocket.connect(oAddr);
MessageEvent * poEvent = new MessageEvent(MessageEventType_SEND, oSocket.detachSocketHandle(),
oAddr, m_poEventLoop, m_poNetWork);
assert(poEvent != nullptr);
m_mapEvent[llNodeID] = poEvent;
m_vecEvent.push_back(poEvent);
PLImp("ok, ip %s port %d", sIP.c_str(), iPort);
return poEvent;
}
void TcpClient :: DealWithWrite()
{
//在每一次epoll_wait结束之后都会打开相应的EPOLLOUT事件,将要发送的数据发送出去。
size_t iSize = m_vecEvent.size();
for (size_t i = 0; i < iSize; i++)
{
m_vecEvent[i]->OpenWrite();
}
//PLImp("end, live event count %zu", vecEventList.size());
}
OK TCP 发送逻辑结束
Phxpaxos网络部分相关的代码逻辑完成。
实现网络部分的基础帮助类没有进行相应的讲解主要在src/utils
目录下面。如Timer,socket等封装。你可以自己看一下,也会有很大的收获。utils相关的测试代码在src/ut
目录下面。