apache hadoop-WorkNote
$wget http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.9.1/hadoop-2.9.1.tar.gz $tar xfz hadoop-2.9.1.tar.gz -C /opt/$cd /opt/
$ ln -s hadoop-2.9.1 hadoop
$ cd hadoop/etc/hadoop
$ grep JAVA_HOME hadoop-env.sh
export JAVA_HOME=/usr/java/latest
$
$ tail -14core-site.xml
fs.defaultFS
hdfs://inspiryhdfs
hadoop.tmp.dir
/opt/hadoop/tmp
ha.zookeeper.quorum
open-stable:2181,permission-stable:2181,sp-stable:2181
$
$ tail -50 hdfs-site.xml
dfs.nameservices
inspiryhdfs
dfs.ha.namenodes.inspiryhdfs
nn1,nn2
dfs.namenode.rpc-address.inspiryhdfs.nn1
oversea-stable:9000
dfs.namenode.http-address.inspiryhdfs.nn1
oversea-stable:50070
dfs.namenode.rpc-address.inspiryhdfs.nn2
bus-stable:9000
dfs.namenode.http-address.inspiryhdfs.nn2
bus-stable:50070
dfs.namenode.shared.edits.dir
qjournal://open-stable:8485;permission-stable:8485;sp-stable:8485/inspiryhdfs
dfs.journalnode.edits.dir
/opt/hadoop/journal
dfs.ha.automatic-failover.enabled
true
dfs.client.failover.proxy.provider.inspiryhdfs
org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
dfs.ha.fencing.methods
sshfence
dfs.ha.fencing.ssh.private-key-files
/home/hadoop/.ssh/id_rsa
指定MapReduce运行在yarn框架之上
$ cp mapred-site.xml{.template,}
$ tail -6mapred-site.xml
mapreduce.framework.name
yarn
$
指定DataNode节点
$ cat slaves
open-stable
permission-stable
sp-stable
$
$ tail -11 yarn-site.xml
yarn.resourcemanager.hostname
oversea-stable
yarn.nodemanager.aux-services
mapreduce_shuffle
$
页:
[1]