“安装hadoop”的版本间的差异
201708010242(讨论 | 贡献) |
|||
第6行: | 第6行: | ||
解压 hadoop 到相应目录: | 解压 hadoop 到相应目录: | ||
+ | |||
+ | [[文件:解压hadoop.png]] | ||
vim /etc/profile | vim /etc/profile | ||
第15行: | 第17行: | ||
export PATH=$PATH:$HADOOP_HOME/bin | export PATH=$PATH:$HADOOP_HOME/bin | ||
+ | [[文件:配置hadoop.png]] | ||
== 编辑 hadoop 环境配置文件 hadoop-env.sh == | == 编辑 hadoop 环境配置文件 hadoop-env.sh == | ||
第20行: | 第23行: | ||
vim hadoop-env.sh | vim hadoop-env.sh | ||
+ | |||
+ | [[文件:修改hadoop-env.png]] | ||
== 修改core-site.xml配置文件 == | == 修改core-site.xml配置文件 == | ||
+ | |||
+ | <nowiki><configuration> | ||
+ | <property> | ||
+ | <name>fs.defaultFS</name> | ||
+ | <value>hdfs://master:9000</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>hadoop.tmp.dir</name> | ||
+ | <value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>io.file.buffer.size</name> | ||
+ | <value>13172</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>dfs.namenode.checkpoint.period</name> | ||
+ | <value>60</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>fs.checkpoint.size</name> | ||
+ | <value>67108864</value> | ||
+ | </property> | ||
+ | </configuration> | ||
+ | </nowiki> | ||
+ | |||
+ | [[文件:配置core-site.png]] | ||
== 修改yarn-site.xml配置文件 == | == 修改yarn-site.xml配置文件 == | ||
+ | |||
+ | <nowiki><configuration> | ||
+ | |||
+ | <property> | ||
+ | <name>yarn.resourcemanager.address</name> | ||
+ | <value>master:18040</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>yarn.resourcemanager.scheduler.address</name> | ||
+ | <value>master:18030</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>yarn.resourcemanager.webapp.address</name> | ||
+ | <value>master:18088</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>yarn.resourcemanager.resource-tracker.address</name> | ||
+ | <value>master:18025</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>yarn.resourcemanager.admin.address</name> | ||
+ | <value>master:18141</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>yarn.nodemanager.aux-services</name> | ||
+ | <value>mapreduce_shuffle</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name> | ||
+ | <value>org.apache.hadoop.mapred.ShuffleHandler</value> | ||
+ | </property> | ||
+ | |||
+ | </configuration> | ||
+ | |||
+ | </nowiki> | ||
+ | |||
== 修改hdfs-site.xml配置文件 == | == 修改hdfs-site.xml配置文件 == | ||
+ | |||
+ | <nowiki> | ||
+ | <configuration> | ||
+ | <property> | ||
+ | <name>dfs.replication</name> | ||
+ | <value>2</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>dfs.namenode.name.dir</name> | ||
+ | <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value> | ||
+ | <final>true</final> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>dfs.datanode.data.dir</name> | ||
+ | <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value> | ||
+ | <final>true</final> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>dfs.namenode.secondary.http-address</name> | ||
+ | <value>master:9001</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>dfs.webhdfs.enabled</name> | ||
+ | <value>true</value> | ||
+ | </property> | ||
+ | <property> | ||
+ | <name>dfs.permissions</name> | ||
+ | <value>false</value> | ||
+ | </property> | ||
+ | </configuration> | ||
+ | |||
+ | </nowiki> | ||
== 修改mapred-site.xml配置文件 == | == 修改mapred-site.xml配置文件 == | ||
+ | <configuration> | ||
+ | <property> | ||
+ | <name>mapreduce.framework.name</name> | ||
+ | <value>yarn</value> | ||
+ | </property> | ||
+ | </configuration> | ||
+ | |||
+ | [[文件:Mapred-site.png]] | ||
== 编写 slavs 文件 == | == 编写 slavs 文件 == | ||
+ | [[文件:修改slaves.png]] | ||
== 编写 master 文件 == | == 编写 master 文件 == | ||
+ | [[文件:修改master.png]] | ||
== 分发 hadoop == | == 分发 hadoop == | ||
+ | scp -r /usr/hadoop root@slave1:/usr/ | ||
+ | scp -r /usr/hadoop root@slave2:/usr/ | ||
== 启动 hadoop 集群 == | == 启动 hadoop 集群 == | ||
+ | strat-dfs.sh | ||
== 浏览中访问 == | == 浏览中访问 == | ||
+ | |||
+ | [[媒体文件:Hadoopweb.png]] | ||
== 查看 hdfs == | == 查看 hdfs == |
2018年11月11日 (日) 09:44的版本
.
目录
安装hadoop,配置环境变量
创建对应工作目录/usr/hadoop:
解压 hadoop 到相应目录:
vim /etc/profile
export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib
export PATH=$PATH:$HADOOP_HOME/bin
编辑 hadoop 环境配置文件 hadoop-env.sh
vim hadoop-env.sh
修改core-site.xml配置文件
<configuration> <property> <name>fs.defaultFS</name> <value>hdfs://master:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value> </property> <property> <name>io.file.buffer.size</name> <value>13172</value> </property> <property> <name>dfs.namenode.checkpoint.period</name> <value>60</value> </property> <property> <name>fs.checkpoint.size</name> <value>67108864</value> </property> </configuration>
修改yarn-site.xml配置文件
<configuration> <property> <name>yarn.resourcemanager.address</name> <value>master:18040</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>master:18030</value> </property> <property> <name>yarn.resourcemanager.webapp.address</name> <value>master:18088</value> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>master:18025</value> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>master:18141</value> </property> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> </configuration>
修改hdfs-site.xml配置文件
<configuration> <property> <name>dfs.replication</name> <value>2</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value> <final>true</final> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value> <final>true</final> </property> <property> <name>dfs.namenode.secondary.http-address</name> <value>master:9001</value> </property> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> </property> <property> <name>dfs.permissions</name> <value>false</value> </property> </configuration>
修改mapred-site.xml配置文件
<configuration>
<property> <name>mapreduce.framework.name</name> <value>yarn</value> </property>
</configuration>
编写 slavs 文件
编写 master 文件
分发 hadoop
scp -r /usr/hadoop root@slave1:/usr/
scp -r /usr/hadoop root@slave2:/usr/
启动 hadoop 集群
strat-dfs.sh