yaomint 发表于 2018-10-28 15:13:35

hadoop基础安装(1)

Hadoop 完全分布式安装-Hadoop—2.7.4  1.获取安装包
  hadoop.apache.com
  2. 配置环境Java环境变量
  2.1 获取jdk安装包
  java.com
  2.2 解压jdk
  $ sudo tar -zxvf jdk1.8.0_144.tar.gz -C /usr/local/
  2.3 配置用户环境变量
  //创建hadoop用户
  $ sudo useradd hadoop
  $ sudo passwd hadoop --密码为hadoop123
  $ su - hadoop
  $ vim ~/.bashrc
  #++++++++++++++++++++++++++++++++++++++++
  export JAVA_HOME=/usr/local/jdk1.8.0_144/
  export JDK_BASH=//usr/local/jdk1.8.0_144/jre

  export>  export PATH=$PATH:$JAVA_HOME/bin:$JDK_BASH/bin
  $ source ~/.bashrc
  $ java -version
  3.配置Hadoop-2.7.4分布式安装
  3.1 解压Hadoop-2.7.4 安装包
  $ tar -zxvf hadoop-2.7.4.tar.gz -C /usr/local/
  3.2 配置Hadoop变量
  $ vim ~/.bashrc
  export HADOOP_HOME=/usr/local/hadoop-2.7.4
  export HADOOP_INSTALL=$HADOOP_HOME
  export HADOOP_MAPRED_HOME=$HADOOP_HOME
  export HADOOP_COMMON_HOME=$HADOOP_HOME
  export HADOOP_HDFS_HOME=$HADOOP_HOME
  export YARN_HOME=$HADOOP_HOME
  export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
  export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:/usr/local/hadoop/bin:/usr/local/hadoop/sbin
  $ source ~/.bashrc
  $ hadoop version
  3.3 配置Hadoop配置文件
  
  3.3.1 配置hdfs的配置文件
  $ cd /usr/local/hadoop-2.7.4/etc/hadoop
  $ vim hdfs-site.xml
  
  
  dfs.permissions.enabled
  false
  
  
  
  dfs.webhdfs.enabled
  true
  
  
  dfs.blocksize
  64m
  
  
  dfs.nameservices
  hadoop-cluster-zgw
  
  
  
  dfs.replication
  2
  
  
  
  dfs.namenode.name.dir
  file:/usr/local/hadoop-2.7.4/hdfs/name
  
  
  
  dfs.namenode.checkpoint.dir
  file:/usr/local/hadoop-2.7.4/hdfs/checkpoint
  
  
  
  dfs.namenode.checkpoint.edits.dir
  file:/usr/local/hadoop-2.7.4/hdfs/checkpoint
  
  
  
  dfs.datanode.data.dir
  file:/usr/local/hadoop-2.7.4/hdfs/data
  
  
  
  dfs.namenode.secondary.http-address
  192.168.240.133:50090
  
  
  3.3.2 配置 yarn的配置文件
  $ vim yarn-site.xml
  
  
  
  yarn.nodemanager.aux-services
  mapreduce_shuffle
  
  
  yarn.nodemanager.aux-services.mapreduce.shuffle.class
  org.apache.hadoop.mapred.ShuffleHandler
  
  
  yarn.nodemanager.local-dirs
  /usr/local/hadoop-2.7.4/yarn/nodemanager
  
  
  yarn.resourcemanager.hostname
  192.168.240.133
  
  
  yarn.resourcemanager.address
  192.168.240.133:8032
  
  
  yarn.resourcemanager.scheduler.address
  192.168.240.133:8030
  
  
  yarn.resourcemanager.resource-tracker.address
  192.168.240.133:8031
  
  
  yarn.resourcemanager.admin.address
  192.168.240.133:8033
  
  
  yarn.resourcemanager.webapp.address
  192.168.240.133:8034
  
  
  3.3.3 配置core
  $ vim core-site.xml
  
  
  fs.defaultFS
  hdfs://192.168.240.133:9000
  
  
  io.file.buffer.size
  131072
  
  
  hadoop.tmp.dir
  file:/usr/local/hadoop-2.7.4/hdfs/tmp
  
  
  3.3.4 配置mapred
  $ vim mapred-site.xml
  
  
  mapreduce.framework.name
  yarn
  
  
  mapreduce.jobhistory.address
  192.168.240.133:10020
  
  
  mapreduce.jobhistory.webapp.address
  192.68.240.133:19888
  
  
  3.3.5 配置slave
  $ vim slaves
  bigdata2
  bigdata3
  4.配置hosts解析和ssh无密码登录
  4.1 配置hosts
  
  $ vim /etc/hosts
  192.168.240.133 bigdata1
  192.168.240.134 bigdata2
  192.168.240.135 bigdata3
  4.2 配置ssh无密码登录
  4.2.1 查看当前用户
  $ whoami
  hadoop
  
  4.2.2 创建ssh的key
  $ ssh-keygen -t rsa
  $ ssh-copy-id -i bigdata1
  $ ssh-copy-id -i bigdata2
  $ ssh-copy-id -i bigdata3
  4.2.3 测试ssh无密码登录
  $ ssh bigdata2
  5.创建配置文件里的目录
  
  $ sudo mkdir -p /usr/local/hadoop-2.7.4/hdfs
  $ sudo mkdir -p /usr/local/hadoop-2.7.4/yarn
  $ sudo mkdir -p /usr/local/hadoop-2.7.4/tmp
  6.拷贝master 节点的Hadoop到其它节点
  $ scp -R hadoop-2.7.4/ bigdata2:/usr/local/
  $ scp -R hadoop-2.7.4/ bigdata3:/usr/local/
  7.修改权限
  
  $sudo chown -R hadoop.hadoop /usr/local/hadoop-2.7.4
  8.启动Hadoop集群
  $ start-all.sh
  

页: [1]
查看完整版本: hadoop基础安装(1)