hadoop节点挂死的一次分析报表。
hadoop的一个节点unused了。然后重启启动hadoop的服务，仍有有一个hadoop的节点起不来。多次重启hadoop和杀进程之后，发现hadoop的master和slave节点上的状态在切换，没有达到同步起停;当master起来的时候，slave节点上的hadoop就unused,当slave节点上的hadoop状态为running的时候，master节点上的hadoop节点的状态就是unused.主从没法同步起停。在数据库入库的时候，日志里报如下错误：
这个时候突然想到了hadoop的主从站点之后的关系是不是没有同步好，文件进入了安全模式。
进入到hadoop的bin目录下，执行退出安全模式的命令，第一次是在hadoop服务停了 之后，执行退出安全模式的命令显示没有连接成功。
然后启动hadoop，这个时候hadoop下面的都运行正常了，执行退出安全模式。
退出安全模式命令： hadoop dfsadmin -safemode leave
]
md64:/usr/lib64:/lib64:/lib:/usr/lib]
]
]
]
]
]
]
]
]
]
] -SendThread(pmweb.site:15248)]
using SASL (unknown error)]
-SendThread(pmweb.site:15248)]
-SendThread(pmweb.site:15248)]
28, negotiated timeout = 180000]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1415)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at com.sun.proxy.$Proxy4.getProtocolVersion(Unknown Source)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:712)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isMasterRunning(HConnectionManager.java:759)
at com.inspur.pm.backend.core.hbase.HbaseFactory.creatTable(HbaseFactory.java:152)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.getFSConnection(FileEngineImplDistributed.java:412)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.insert2HbaseAndOracle(FileEngineImplDistributed.java:744)
at com.inspur.pm.backend.core.fileengine.impl.FileEngineImplDistributed.putFile2FSByFileDesc(FileEngineImplDistributed.java:275)
at com.inspur.pmv5.dpl.pif2pia.service.CommonService.storeFileToHBASE(CommonService.java:97)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.storePIA2HBASE(BussinessLogicDealMain.java:1115)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.generatePIAFile(BussinessLogicDealMain.java:610)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessLogicDealMain.deal(BussinessLogicDealMain.java:414)
at com.inspur.pmv5.dpl.pif2pia.service.BussinessThread.run(BussinessThread.java:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
]
页:
[1]