上篇相关博客：Zookeeper节点及客户端基本操作
Zookeeper服务端的三种启动模式
1）standalone，单机模式;2）伪分布式模式（单机模拟集群）；3）分布式模式（集群模式）。不管哪种模式启动服务，启动类都是org.apache.zookeeper.server.quorum.QuorumPeerMain，本文主要介绍的是单机模式启动流程。以下提到的源码都只是其中部分关键源码，如需要查看完整源码请移步：https://github.com/qqxhb/zookeeper
1. org.apache.zookeeper.server.quorum.QuorumPeerMain
public class QuorumPeerMain {
private static final Logger LOG = LoggerFactory.getLogger(QuorumPeerMain.class);
private static final String USAGE = "Usage: QuorumPeerMain configfile";
protected QuorumPeer quorumPeer;
/**
* To start the replicated server specify the configuration file name on
* the command line.
* @param args path to the configfile
*/
public static void main(String[] args) {
//构建QuorumPeerMain实例
QuorumPeerMain main = new QuorumPeerMain();
try {
//主要逻辑就一个初始化方法
main.initializeAndRun(args);
//省略其他异常判断
} catch (Exception e) {
LOG.error("Unexpected exception, exiting abnormally", e);
System.exit(1);
}
LOG.info("Exiting normally");
System.exit(0);
}
protected void initializeAndRun(String[] args)
throws ConfigException, IOException, AdminServerException
{
//创建配置类实例
QuorumPeerConfig config = new QuorumPeerConfig();
if (args.length == 1) {
//解析配置，见步骤2
config.parse(args[0]);
}
// 创建并启动快照、事务历史文件清理器DatadirCleanupManager
DatadirCleanupManager purgeMgr = new DatadirCleanupManager(config
.getDataDir(), config.getDataLogDir(), config
.getSnapRetainCount(), config.getPurgeInterval());
purgeMgr.start();
//判断是否是集群模式：如果servers集合中的元素个数大于0，则是集群模式，否则是单机模式。
if (args.length == 1 && config.isDistributed()) {
runFromConfig(config);
} else {
LOG.warn("Either no config or no quorum defined in config, running "
+ " in standalone mode");
//如果是单机模式，则委托给ZookeeperServerMain进行处理。
ZooKeeperServerMain.main(args);
}
}
2. 装载并解析配置org.apache.zookeeper.server.quorum.QuorumPeerConfig
当中涉及的配置含义请参考Zookeeper入门及单机及集群环境搭建的配置部分
public void parse(String path) throws ConfigException {
LOG.info("Reading configuration from: " + path);
try {
File configFile = (new VerifyingFileFactory.Builder(LOG)
.warnForRelativePath()
.failForNonExistingPath()
.build()).create(path);
//将指定 的配置文件加载到 Properties
Properties cfg = new Properties();
FileInputStream in = new FileInputStream(configFile);
try {
cfg.load(in);
configFileStr = path;
} finally {
in.close();
}
//将Properties转换成QuorumPeerConfig
//通过可以获取Properties中的属性，简单处理或者判断赋值给QuorumPeerConfig中的属性
parseProperties(cfg);
} catch (IOException e) {
throw new ConfigException("Error processing " + path, e);
} catch (IllegalArgumentException e) {
throw new ConfigException("Error processing " + path, e);
}
//省略动态配置部分源码
}
3. org.apache.zookeeper.server.DatadirCleanupManager
在第一步的main方法加载配置之后，会根据配置的数据日志路径及清理周期创建DatadirCleanupManager实例，并启动定时清理任务
public class DatadirCleanupManager {
private static final Logger LOG = LoggerFactory.getLogger(DatadirCleanupManager.class);
/**
* 清洗任务状态
*/
public enum PurgeTaskStatus {
NOT_STARTED, STARTED, COMPLETED;
}
private PurgeTaskStatus purgeTaskStatus = PurgeTaskStatus.NOT_STARTED;
//快照（数据）路径
private final File snapDir;
//日志（数据）路径
private final File dataLogDir;
//这个参数和下面的参数搭配使用，这个参数指定了需要保留的文件数目。默认是保留3个，也是3.4以后才有的。
private final int snapRetainCount;
//3.4.0及之后版本，ZK提供了自动清理事务日志和快照文件的功能，这个参数指定了清理频率，单位是小时，需要配置一个1或更大的整数，默认是0，表示不开启自动清理功能。
private final int purgeInterval;
//TImer定时器，根据上面的配置定时执行清理
private Timer timer;
//省略业务逻辑
}
4. org.apache.zookeeper.server.ZooKeeperServerMain
步骤1提到如果是单机模式则会委托给ZooKeeperServerMain执行：
org.apache.zookeeper.server.ZooKeeperServerMain.main(String[]) ——>org.apache.zookeeper.server.ZooKeeperServerMain.initializeAndRun(String[])解析配置并运行服务
protected void initializeAndRun(String[] args)
throws ConfigException, IOException, AdminServerException
{
try {
//注册Log4JBean，可以设置zookeeper.jmx.log4j.disable=true，禁用
ManagedUtil.registerLog4jMBeans();
} catch (JMException e) {
LOG.warn("Unable to register log4j JMX control", e);
}
//创建并ServerConfig实例并解析配置文件
ServerConfig config = new ServerConfig();
if (args.length == 1) {
//如果只有一个参数，则认为是指定的配置文件，会调用QuorumPeerConfig的parse方法，然后在复制到ServerConfig的属性中
config.parse(args[0]);
} else {
//否则认为属性直接是参数形式传入clientPortAddress、dataDir/dataLogDir、tickTime、maxClientCnxns
//前面两个端口和路径参数是必传，后两个可不传
config.parse(args);
}
//根据配置文件内容 运行服务
runFromConfig(config);
}
5. org.apache.zookeeper.server.ServerConfig
public class ServerConfig {
// 客户端连接端口
protected InetSocketAddress clientPortAddress;
// 客户端安全连接端口
protected InetSocketAddress secureClientPortAddress;
// 数据路径
protected File dataDir;
// 日志数据路径
protected File dataLogDir;
//# 这个时间是作为 Zookeeper心跳时间单位（如5表示5毫秒，若心跳间隔配置的是10，则表示5*10=50毫秒同步一次心跳）。
protected int tickTime = ZooKeeperServer.DEFAULT_TICK_TIME;//默认3000
/*
* 单个客户端与单台服务器之间的连接数的限制，是ip级别的，默认是60，如果设置为0，那么表明不作任何限制。
* 请注意这个限制的使用范围，仅仅是单台客户端机器与单台ZK服务器之间的连接数限制，不是针对指定客户端IP，也不是ZK集群的连接数限制，
* 也不是单台ZK对所有客户端的连接数限制。
*/
protected int maxClientCnxns;
/** 最小会话超时时间，默认-1表示永久 */
protected int minSessionTimeout = -1;
/** 最大会话超时时间*/
protected int maxSessionTimeout = -1;
//省略内部逻辑
}
6.org.apache.zookeeper.server.ZooKeeperServerMain.runFromConfig(ServerConfig)
服务启动的主要流程都在这个方法中
public void runFromConfig(ServerConfig config)
throws IOException, AdminServerException {
LOG.info("Starting server");
FileTxnSnapLog txnLog = null;
try {
// 创建事务及快照日志文件管理实例
txnLog = new FileTxnSnapLog(config.dataLogDir, config.dataDir);
//创建ZooKeeperServer实例
final ZooKeeperServer zkServer = new ZooKeeperServer(txnLog,
config.tickTime, config.minSessionTimeout, config.maxSessionTimeout, null);
//给txnLog 设置服务统计实例
txnLog.setServerStats(zkServer.serverStats());
//注册服务异常或关闭事件处理类，服务启动完成后会调用之后shutdownLatch.await();让主线程一直阻塞，
//直到ZooKeeperServerShutdownHandler的handle方法调用 shutdownLatch.countDown();
final CountDownLatch shutdownLatch = new CountDownLatch(1);
zkServer.registerServerShutdownHandler(
new ZooKeeperServerShutdownHandler(shutdownLatch));
// 启动Adminserver，默认是JettyAdminServer，启动之后可以访问服务的状态信息 http://localhost:8080/commands
adminServer = AdminServerFactory.createAdminServer();
adminServer.setZooKeeperServer(zkServer);
adminServer.start();
/*
* ServerCnxnFactory是Zookeeper中的重要组件,负责处理客户端与服务器的连接.主要有两个实现,
* 一个是NIOServerCnxnFactory,使用Java原生NIO处理网络IO事件;
* 另一个是NettyServerCnxnFactory,使用Netty处理网络IO事件.作为处理客户端连接的组件,其会启动若干线程监听客户端连接端口(即默认的9876端口)
*/
boolean needStartZKServer = true;
//创建并启动CnxnFactory，负责处理客户端与服务器的连接，默认是NIOServerCnxnFactory，可以通过系统配置zookeeper.serverCnxnFactory指定
if (config.getClientPortAddress() != null) {
cnxnFactory = ServerCnxnFactory.createFactory();
cnxnFactory.configure(config.getClientPortAddress(), config.getMaxClientCnxns(), false);
cnxnFactory.startup(zkServer);
// 启动CnxnFactory是自动了ZKServer则不需要再secureCnxnFactory中再启动了
needStartZKServer = false;
}
//创建并启动secureCnxnFactory，默认是NIOServerCnxnFactory，可以通过系统配置zookeeper.serverCnxnFactory指定
if (config.getSecureClientPortAddress() != null) {
secureCnxnFactory = ServerCnxnFactory.createFactory();
secureCnxnFactory.configure(config.getSecureClientPortAddress(), config.getMaxClientCnxns(), true);
secureCnxnFactory.startup(zkServer, needStartZKServer);
}
//创建并启动ContainerManager
containerManager = new ContainerManager(zkServer.getZKDatabase(), zkServer.firstProcessor,
Integer.getInteger("znode.container.checkIntervalMs", (int) TimeUnit.MINUTES.toMillis(1)),
Integer.getInteger("znode.container.maxPerMinute", 10000)
);
containerManager.start();
//主线程阻塞，等待shutdownHandler的handel方法被调用，结束阻塞
shutdownLatch.await();
//执行shutdown逻辑，关闭containerManager、cnxnFactory、adminServer等
shutdown();
if (cnxnFactory != null) {
cnxnFactory.join();
}
if (secureCnxnFactory != null) {
secureCnxnFactory.join();
}
// 关闭zkserver
if (zkServer.canShutdown()) {
zkServer.shutdown(true);
}
} catch (InterruptedException e) {
// warn, but generally this is ok
LOG.warn("Server interrupted", e);
} finally {
if (txnLog != null) {
txnLog.close();
}
}
}
7. 创建ZooKeeperServer实例的时候会创建ServerStats和ZooKeeperServerListener实例
public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime,
int minSessionTimeout, int maxSessionTimeout, ZKDatabase zkDb) {
//创建ServerStats实例
serverStats = new ServerStats(this);
this.txnLogFactory = txnLogFactory;
this.txnLogFactory.setServerStats(this.serverStats);
this.zkDb = zkDb;
this.tickTime = tickTime;
setMinSessionTimeout(minSessionTimeout); // 最小的sessionTimeout，默认为tickTime * 2
setMaxSessionTimeout(maxSessionTimeout); // 最大的sessionTimeout，默认为tickTime * 20
// 如果关键线程发生错误而停止了，那么就通过这个监听器修改zkServer的状态为ERROR.
listener = new ZooKeeperServerListenerImpl(this);
LOG.info("Created server with tickTime " + tickTime
+ " minSessionTimeout " + getMinSessionTimeout()
+ " maxSessionTimeout " + getMaxSessionTimeout()
+ " datadir " + txnLogFactory.getDataDir()
+ " snapdir " + txnLogFactory.getSnapDir());
}
8. org.apache.zookeeper.server.persistence.FileTxnSnapLog主要属性及方法
public class FileTxnSnapLog {
//事务日志路径
private final File dataDir;
//快照数据路径
private final File snapDir;
//负责处理快照日志
private TxnLog txnLog;
//负责处理事务日志
private SnapShot snapLog;
//省略其他操作快照和事务日志逻辑
/**
*启动ZookeeperServer时调用此方法从磁盘上的快照和事务日志中恢复数据
*/
public long restore(DataTree dt, Map<Long, Integer> sessions,
PlayBackListener listener) throws IOException {
long deserializeResult = snapLog.deserialize(dt, sessions);
FileTxnLog txnLog = new FileTxnLog(dataDir);
if (-1L == deserializeResult) {
/* this means that we couldn't find any snapshot, so we need to
* initialize an empty database (reported in ZOOKEEPER-2325) */
if (txnLog.getLastLoggedZxid() != -1) {
// ZOOKEEPER-3056: provides an escape hatch for users upgrading
// from old versions of zookeeper (3.4.x, pre 3.5.3).
if (!trustEmptySnapshot) {
throw new IOException(EMPTY_SNAPSHOT_WARNING + "Something is broken!");
} else {
LOG.warn(EMPTY_SNAPSHOT_WARNING + "This should only be allowed during upgrading.");
}
}
/* TODO: (br33d) we should either put a ConcurrentHashMap on restore()
* or use Map on save() */
save(dt, (ConcurrentHashMap<Long, Integer>)sessions);
/* return a zxid of zero, since we the database is empty */
return 0;
}
return fastForwardFromEdits(dt, sessions, listener);
}
/**
* 获取日志中记载的最新的zxid
*/
public long getLastLoggedZxid() {
FileTxnLog txnLog = new FileTxnLog(dataDir);
return txnLog.getLastLoggedZxid();
}
/**
*将内存中的数据持久化到磁盘中
*/
public void save(DataTree dataTree,
ConcurrentHashMap<Long, Integer> sessionsWithTimeouts)
throws IOException {
long lastZxid = dataTree.lastProcessedZxid;
File snapshotFile = new File(snapDir, Util.makeSnapshotName(lastZxid));
LOG.info("Snapshotting: 0x{} to {}", Long.toHexString(lastZxid),
snapshotFile);
snapLog.serialize(dataTree, sessionsWithTimeouts, snapshotFile);
}
}
9. AdminServer 界面
9. 启动网络IO管理器CnxnFactory和ZooKeeperServer org.apache.zookeeper.server.NIOServerCnxnFactory.startup(ZooKeeperServer, boolean)
public void startup(ZooKeeperServer zks, boolean startServer)
throws IOException, InterruptedException {
/*
* 启动选举线程 selectorThreads
* 启动接收线程 AcceptThread
* 启动连接超时检测线程 ConnectionExpirerThread
*/
start();
//设置ZooKeeperServer
setZooKeeperServer(zks);
//判断是否需要启动ZooKeeperServer
if (startServer) {
//Zookeeper启动的时候，需要从本地快照数据文件和事务日志文件中进行数据恢复
zks.startdata();
//创建并启动会话管理器、初始化Zookeeper的请求处理链、注册JMX
zks.startup();
}
}
10. org.apache.zookeeper.server.ZooKeeperServer.startup()
public synchronized void startup() {
if (sessionTracker == null) {
createSessionTracker();
}
// 会创建一个会话管理器SessionTracker。
// 创建SessionTracker时会初始化expirationInterval、nextExpirationTime和sessionWithTimeout，同时还会计算出一个初始化的sessionId
startSessionTracker();
// 初始化Zookeeper的请求处理链
setupRequestProcessors();
// 注册JMX服务,Zookeeper会将服务器运行时的一些信息以JMX的方式暴露给外部
registerJMX();
// 设置服务为运行状态
setState(State.RUNNING);
notifyAll();
}
/**
* Zookeeper的请求处理方式是典型的责任链模式的实现，在Zookeeper服务器上，会有多个请求处理器依次处理一个客户端请求。
* 在服务器启动的时候，会将这些请求处理器串联起来形成一个处理器链。处理链依次为： PreRequestProcessor ——>
* SyncRequestProcessor ——> FinalRequestProcessor
*/
protected void setupRequestProcessors() {
RequestProcessor finalProcessor = new FinalRequestProcessor(this);
RequestProcessor syncProcessor = new SyncRequestProcessor(this, finalProcessor);
((SyncRequestProcessor) syncProcessor).start();
firstProcessor = new PrepRequestProcessor(this, syncProcessor);
((PrepRequestProcessor) firstProcessor).start();
}
11. org.apache.zookeeper.server.PrepRequestProcessor
主要负责接收客户端请求校验ACL、生成事务及节点修改记录
org.apache.zookeeper.server.PrepRequestProcessor.processRequest(Request)将请求添加到org.apache.zookeeper.server.PrepRequestProcessor.submittedRequests队列中，然后改线程的run方法会不断的去取出请求submittedRequests.take();，再由org.apache.zookeeper.server.PrepRequestProcessor.pRequest(Request)方法根据不同的类型，构造不同的请求实例调用不同的处理方法，比如create会构建CreateRequest实例，然后调用org.apache.zookeeper.server.PrepRequestProcessor.pRequest2Txn(int, long, Request, Record, boolean)处理 ——> org.apache.zookeeper.server.PrepRequestProcessor.pRequest2TxnCreate(int, Request, Record, boolean)
private void pRequest2TxnCreate(int type, Request request, Record record, boolean deserialize) throws IOException, KeeperException {
if (deserialize) {
ByteBufferInputStream.byteBuffer2Record(request.request, record);
}
int flags;
String path;
List<ACL> acl;
byte[] data;
long ttl;
if (type == OpCode.createTTL) {
CreateTTLRequest createTtlRequest = (CreateTTLRequest)record;
flags = createTtlRequest.getFlags();
path = createTtlRequest.getPath();
acl = createTtlRequest.getAcl();
data = createTtlRequest.getData();
ttl = createTtlRequest.getTtl();
} else {
CreateRequest createRequest = (CreateRequest)record;
flags = createRequest.getFlags();
path = createRequest.getPath();
acl = createRequest.getAcl();
data = createRequest.getData();
ttl = -1;
}
//构建校验请求
CreateMode createMode = CreateMode.fromFlag(flags);
//校验创建请求
validateCreateRequest(path, createMode, request, ttl);
//获取父节点路径
String parentPath = validatePathForCreate(path, request.sessionId);
//解析ACL
List<ACL> listACL = fixupACL(path, request.authInfo, acl);
//获取父节点信息
ChangeRecord parentRecord = getRecordForPath(parentPath);
//校验ACL
checkACL(zks, parentRecord.acl, ZooDefs.Perms.CREATE, request.authInfo);
int parentCVersion = parentRecord.stat.getCversion();
//如果是顺序节点，则需要重新定义路径
if (createMode.isSequential()) {
path = path + String.format(Locale.ENGLISH, "%010d", parentCVersion);
}
//节点路径校验
validatePath(path, request.sessionId);
try {
//判断是否已经存在该路路径节点
if (getRecordForPath(path) != null) {
throw new KeeperException.NodeExistsException(path);
}
} catch (KeeperException.NoNodeException e) {
// ignore this one
}
//临时节点校验
boolean ephemeralParent = EphemeralType.get(parentRecord.stat.getEphemeralOwner()) == EphemeralType.NORMAL;
if (ephemeralParent) {
throw new KeeperException.NoChildrenForEphemeralsException(path);
}
int newCversion = parentRecord.stat.getCversion()+1;
//根据创建类型生成不同的Txn事务
if (type == OpCode.createContainer) {
request.setTxn(new CreateContainerTxn(path, data, listACL, newCversion));
} else if (type == OpCode.createTTL) {
request.setTxn(new CreateTTLTxn(path, data, listACL, newCversion, ttl));
} else {
request.setTxn(new CreateTxn(path, data, listACL, createMode.isEphemeral(),
newCversion));
}
StatPersisted s = new StatPersisted();
if (createMode.isEphemeral()) {
s.setEphemeralOwner(request.sessionId);
}
//更新父节点和子节点相关的信息
parentRecord = parentRecord.duplicate(request.getHdr().getZxid());
parentRecord.childCount++;
parentRecord.stat.setCversion(newCversion);
//添加修改记录到outstandingChanges队列
addChangeRecord(parentRecord);
addChangeRecord(new ChangeRecord(request.getHdr().getZxid(), path, s, 0, listACL));
}
12. org.apache.zookeeper.server.SyncRequestProcessor
org.apache.zookeeper.server.SyncRequestProcessor.processRequest(Request)将请求添加到org.apache.zookeeper.server.SyncRequestProcessor.queuedRequests队列中，线程的run()从队列取出请求，将Txn事务持久化到磁盘然后进行快照，结束之后调用下一个处理类（FinalRequestProcessor）
13. org.apache.zookeeper.server.FinalRequestProcessor
processRequest方法从org.apache.zookeeper.server.ZooKeeperServer.outstandingChanges获取请求，根据请求更新DataTree、触发Watcher及构造Response返回给客户端
