Hadoop 集群部署

Hadoop环境部署

首先将hadoop上传到linux的/usr/hadoop目录下解压

tar -zxvf hadoop.tar.gz -C /usr/hadoop

配置环境变量

export HADOOP_HOME=/
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

hadoop配置文件

hadoop-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_171

core-site.xml

<property><name>fs.default.name</name><value>hdfs://master:9000</value></property>
<property><name>hadoop.tmp.dir</name><value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value></property>
<property><name>io.file.buffer.size</name><value>131072</value></property>
<property><name>fs.checkpoint.period</name><value>60</value></property>
<property><name>fs.checkpoint.size</name><value>67108864</value></property>

hdfs-site.xml

<property><name>dfs.replication</name><value>2</value></property>
<property><name>dfs.namenode.name.dir</name><value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value></property>
<property><name>dfs.datanode.data.dir</name><value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value></property>

yarn-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_171

yarn-site.xml

<property><name>yarn.resourcemanager.address</name><value>master:18040</value></property>
<property><name>yarn.resourcemanager.scheduler.address</name><value>master:18030</value></property>
<property><name>yarn.resourcemanager.webapp.address</name><value>master:18088</value></property>
<property><name>yarn.resourcemanager.resource-tracker.address</name><value>master:18025</value></property>
<property><name>yarn.resourcemanager.admin.address</name><value>master:18141</value></property>
<property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property>
<property><name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property>

mapred-site.xml

<property><name>mapreduce.framework.name</name><value>yarn</value></property>

master

master

slave

slave1
slave2

格式化namenode：

hadoop namenode -format

启动集群：

start-all.sh