1.基础环境

OS： CentOS 7Hadoop：3.0.1java：1.8.0hadoop master 一台hadoop slave 两台master和slave必须完全相互通信

2. Java 安装

> wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2F; oraclelicense=accept-securebackup-cookie" "; rpm -ivh jdk-8u161-linux-x64.rpm# 配置Java_HOME> vim /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_161/export JRE_HOME=/usr/java/jdk1.8.0_161/jreexport PATH=$PATH:$JAVA_HOME/binexport CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar> source /etc/profile

3. hadoop master 安装及配置

# 创建hadoop 用户

> useradd hadoop> su - hadoop# 生成ssh公私钥对并给三台机器的hadoop用户部署上> ssh-keygen> cp -ar /home/hadoop/.ssh/id_rsa.pub /home/hadoop/.ssh/authorized_keys

# 安装hadoop

> wget ; tar zxvf hadoop-3.0.1.tar.gz -C /usr/local> cd /usr/local/ && mv hadoop-3.0.1 hadoop> chown hadoop.hadoop -R /usr/local/hadoop # 配置hadoop 环境变量> vim /etc/profileexport HADOOP_HOME=/usr/local/hadoopexport PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinexport HADOOP_PREFIX=/usr/local/hadoopexport HADOOP_HDFS_HOME=/usr/local/hadoopexport HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop> source /etc/profile

# 配置hadoop（master server 上执行，请使用hadoop 用户操作）

# core-site.xml> cat /usr/local/hadoop/etc/hadoop/core-site.xml <configuration> <property> <name>fs.defaultFS</name> <value>hdfs://hadoop-master:9000</value> </property> </configuration># hdfs-site.xml (这里要先在三台机器上以hadoop用户创建好目录/data/hadoop/namenode /data/hadoop/datanode)> cat /usr/loca/hadoop/etc/hadoop/hdfs-site.xml <configuration> <property> <name>dfs.replication</name> <value>2</value> <!-- 表示每份数据有两份副本 --> </property> <property> <name>dfs.namenode.name.dir</name> <value>/data/hadoop/namenode</value> <!-- namenode的数据保存目录 --> </property> <property> <name>dfs.datanode.data.dir</name> <value>/data/hadoop/datanode</value> <!-- datanode的数据保存目录 --> </property> </configuration># yarn-site.xml> cat /usr/local/hadoop/etc/hadoop/yarn-site.xml <configuration> <property> <name>yarn.resourcemanager.hostname</name> <value>hadoop-master</value> </property> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.env-whitelist</name> <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value> </property> <property> <name>yarn.nodemanager.vmem-check-enabled</name> <value>false</value> </property> <property> <name>yarn.nodemanager.resource.memory-mb</name> <value>49152</value> </property> <property> <name>yarn.scheduler.maximum-allocation-mb</name> <value>49152</value> </property> </configuration># mapred-site.xml> cat /usr/local/hadoop/etc/hadoop/mapred-site.xml <configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> </configuration> # workers (注意，这里改成了workers文件, 在3.0以前的版本中此文件名为slaves)> cat /usr/local/hadoop/etc/hadoop/workers  hadoop-slave-1 hadoop-slave-2

4. hadoop slave 安装及配置

按照 master 的安装及配置方法部署slave 服务器

注：如果使用的公有云服务器，可以对master制作镜像，然后通过镜像启动slave。

5.hadoop 初始化

# 配置master 和 slave server hosts文件

> cat /etc/hosts1.1.1.1 hadoop-master2.2.2.2 hadoop-slave-13.3.3.3 hadoop-slave-2

# 格式化hdfs(使用hadoop用户)

> hdfs namenode -format

# 启动hdfs(只在master机器上以hadoop用户执行，master会通过ssh将slave上的datanode进程启动)

> start-dfs.sh # 启动 namenode、secondarynamenode、datanode

# 启动yarn

> start-yarn.sh # 启动ResourceManager、NodeManager

# 检查进程

# master> jps14739 ResourceManager14165 NameNode14390 SecondaryNameNode30666 Jps# slave> jps9672 DataNode9931 NodeManage

# web访问hadoop

master-server-ip:8088

web 访问Hadoop master

# 使用命令行查看hdfs(可以任何一台机器执行)

hdfs dfs -ls /