最近启动 Hadoop， nodemanger 老挂，报kerberos 验证错误，各种查找原因，时间也同步，kint 也能登录到kerberos，一直找不到原因，最后发现是网关和远端的时间同步，但是不在同一个时区导致的问题
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.io.IOException: Failed on
local
exception: java.io.IOException: Couldn't setup connection for single/[email protected] to
0.0
.
0.0
/
0.0
.
0.0:
8031
; Host Details :
local
host is:
"remote.bdsm.cmcc/10.2.41.15"
; destination host is:
"0.0.0.0"
:
8031
;
at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:
197
)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:
193
)
at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:
120
)
at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceStart(NodeManager.java:
264
)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:
193
)
at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:
463
)
at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:
509
)
Caused by: java.io.IOException: Failed on
local
exception: java.io.IOException: Couldn't setup connection for single/[email protected] to
0.0
.
0.0
/
0.0
.
0.0:
8031
; Host Details :
local
host is:
"remote.bdsm.cmcc/10.2.41.15"
; destination host is:
"0.0.0.0"
:
8031
;
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:
772
)
at org.apache.hadoop.ipc.Client.call(Client.java:
1472
)
at org.apache.hadoop.ipc.Client.call(Client.java:
1399
)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:
232
)
at com.sun.proxy.$Proxy
28
.registerNodeManager(Unknown Source)
at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:
68
)
at sun.reflect.NativeMethodAccessorImpl.invoke
0
(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
57
)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:
43
)
at java.lang.reflect.Method.invoke(Method.java:
606
)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:
187
)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:
102
)
at com.sun.proxy.$Proxy
29
.registerNodeManager(Unknown Source)
at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:
257
)
at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:
191
)
...
6
more
Caused by: java.io.IOException: Couldn't setup connection for single/[email protected] to
0.0
.
0.0
/
0.0
.
0.0:
8031
at org.apache.hadoop.ipc.Client$Connection$
1
.run(Client.java:
672
)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:
415
)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:
1628
)
at org.apache.hadoop.ipc.Client$Connection.handleSaslConnectionFailure(Client.java:
643
)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:
730
)
at org.apache.hadoop.ipc.Client$Connection.access$
2800
(Client.java:
368
)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:
1521
)
at org.apache.hadoop.ipc.Client.call(Client.java:
1438
)
...
19
more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Clock skew too great (
37
) - PROCESS_TGS)]
at com.sun.security.sasl.gsskerb.GssKrb
5
Client.evaluateChallenge(GssKrb
5
Client.java:
212
)
at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:
413
)
at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:
553
)
at org.apache.hadoop.ipc.Client$Connection.access$
1800
(Client.java:
368
)
at org.apache.hadoop.ipc.Client$Connection$
2
.run(Client.java:
722
)
at org.apache.hadoop.ipc.Client$Connection$
2
.run(Client.java:
718
)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:
415
)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:
1628
)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:
717
)
...
22
more
Caused by: GSSException: No valid credentials provided (Mechanism level: Clock skew too great (
37
) - PROCESS_TGS)
at sun.security.jgss.krb
5
.Krb
5
Context.initSecContext(Krb
5
Context.java:
710
)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:
248
)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:
179
)
at com.sun.security.sasl.gsskerb.GssKrb
5
Client.evaluateChallenge(GssKrb
5
Client.java:
193
)
...
31
more
Caused by: KrbException: Clock skew too great (
37
) - PROCESS_TGS
at sun.security.krb
5
.KrbTgsRep.<init>(KrbTgsRep.java:
73
)
at sun.security.krb
5
.KrbTgsReq.getReply(KrbTgsReq.java:
192
)
at sun.security.krb
5
.KrbTgsReq.sendAndGetCreds(KrbTgsReq.java:
203
)
at sun.security.krb
5
.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:
309
)
at sun.security.krb
5
.internal.CredentialsUtil.acquireServiceCreds(CredentialsUtil.java:
115
)
at sun.security.krb
5
.Credentials.acquireServiceCreds(Credentials.java:
454
)
at sun.security.jgss.krb
5
.Krb
5
Context.initSecContext(Krb
5
Context.java:
641
)
...
34
more
Caused by: KrbException: Identifier doesn't match expected value (
906
)
at sun.security.krb
5
.internal.KDCRep.init(KDCRep.java:
143
)
at sun.security.krb
5
.internal.TGSRep.init(TGSRep.java:
66
)
at sun.security.krb
5
.internal.TGSRep.<init>(TGSRep.java:
61
)
at sun.security.krb
5
.KrbTgsRep.<init>(KrbTgsRep.java:
55
)
...
40
more
|
问题解决：
1. 查看两台当前时区
命令 ： "date -R"
2. 修改设置Linux服务器时区
命令 ： "tzselect"
3. 复制相应的时区文件，替换系统时区文件；或者创建链接文件
cp /usr/share/zoneinfo/$主时区/$次时区 /etc/localtime
例如：在设置中国时区使用亚洲/上海（+8）
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.io.IOException: Failed on
local
exception: java.io.IOException: Couldn't setup connection for single/[email protected] to
0.0
.
0.0
/
0.0
.
0.0:
8031
; Host Details :
local
host is:
"remote.bdsm.cmcc/10.2.41.15"
; destination host is:
"0.0.0.0"
:
8031
;
at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:
197
)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:
193
)
at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:
120
)
at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceStart(NodeManager.java:
264
)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:
193
)
at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:
463
)
at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:
509
)
Caused by: java.io.IOException: Failed on
local
exception: java.io.IOException: Couldn't setup connection for single/[email protected] to
0.0
.
0.0
/
0.0
.
0.0:
8031
; Host Details :
local
host is:
"remote.bdsm.cmcc/10.2.41.15"
; destination host is:
"0.0.0.0"
:
8031
;
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:
772
)
at org.apache.hadoop.ipc.Client.call(Client.java:
1472
)
at org.apache.hadoop.ipc.Client.call(Client.java:
1399
)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:
232
)
at com.sun.proxy.$Proxy
28
.registerNodeManager(Unknown Source)
at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:
68
)
at sun.reflect.NativeMethodAccessorImpl.invoke
0
(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
57
)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:
43
)
at java.lang.reflect.Method.invoke(Method.java:
606
)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:
187
)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:
102
)
at com.sun.proxy.$Proxy
29
.registerNodeManager(Unknown Source)
at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:
257
)
at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:
191
)
...
6
more
Caused by: java.io.IOException: Couldn't setup connection for single/[email protected] to
0.0
.
0.0
/
0.0
.
0.0:
8031
at org.apache.hadoop.ipc.Client$Connection$
1
.run(Client.java:
672
)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:
415
)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:
1628
)
at org.apache.hadoop.ipc.Client$Connection.handleSaslConnectionFailure(Client.java:
643
)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:
730
)
at org.apache.hadoop.ipc.Client$Connection.access$
2800
(Client.java:
368
)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:
1521
)
at org.apache.hadoop.ipc.Client.call(Client.java:
1438
)
...
19
more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Clock skew too great (
37
) - PROCESS_TGS)]
at com.sun.security.sasl.gsskerb.GssKrb
5
Client.evaluateChallenge(GssKrb
5
Client.java:
212
)
at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:
413
)
at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:
553
)
at org.apache.hadoop.ipc.Client$Connection.access$
1800
(Client.java:
368
)
at org.apache.hadoop.ipc.Client$Connection$
2
.run(Client.java:
722
)
at org.apache.hadoop.ipc.Client$Connection$
2
.run(Client.java:
718
)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:
415
)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:
1628
)
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:
717
)
...
22
more
Caused by: GSSException: No valid credentials provided (Mechanism level: Clock skew too great (
37
) - PROCESS_TGS)
at sun.security.jgss.krb
5
.Krb
5
Context.initSecContext(Krb
5
Context.java:
710
)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:
248
)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:
179
)
at com.sun.security.sasl.gsskerb.GssKrb
5
Client.evaluateChallenge(GssKrb
5
Client.java:
193
)
...
31
more
Caused by: KrbException: Clock skew too great (
37
) - PROCESS_TGS
at sun.security.krb
5
.KrbTgsRep.<init>(KrbTgsRep.java:
73
)
at sun.security.krb
5
.KrbTgsReq.getReply(KrbTgsReq.java:
192
)
at sun.security.krb
5
.KrbTgsReq.sendAndGetCreds(KrbTgsReq.java:
203
)
at sun.security.krb
5
.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:
309
)
at sun.security.krb
5
.internal.CredentialsUtil.acquireServiceCreds(CredentialsUtil.java:
115
)
at sun.security.krb
5
.Credentials.acquireServiceCreds(Credentials.java:
454
)
at sun.security.jgss.krb
5
.Krb
5
Context.initSecContext(Krb
5
Context.java:
641
)
...
34
more
Caused by: KrbException: Identifier doesn't match expected value (
906
)
at sun.security.krb
5
.internal.KDCRep.init(KDCRep.java:
143
)
at sun.security.krb
5
.internal.TGSRep.init(TGSRep.java:
66
)
at sun.security.krb
5
.internal.TGSRep.<init>(TGSRep.java:
61
)
at sun.security.krb
5
.KrbTgsRep.<init>(KrbTgsRep.java:
55
)
...
40
more
|
问题解决：
1. 查看两台当前时区
命令 ： "date -R"
2. 修改设置Linux服务器时区
命令 ： "tzselect"
3. 复制相应的时区文件，替换系统时区文件；或者创建链接文件
cp /usr/share/zoneinfo/$主时区/$次时区 /etc/localtime
例如：在设置中国时区使用亚洲/上海（+8）
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime