hadoop文件配置

小程序：扫一扫查出行
【扫一扫了解最新限行尾号】
复制小程序

伪分布式配置：

core-site.xml

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>

hdfs-site.xml

<configuration>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/hadoop/hadoop_data/hdfs/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop/hadoop_data/hdfs/datanode</value>
</property>
</configuration>

mapred-site.xml (可能需要重命名mapred-site.xml.template)

cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>

yarn-site.xml

初始化

mkdir -p /usr/local/hadoop/hadoop_data/hdfs/namenode
mkdir -p /usr/local/hadoop/hadoop_data/hdfs/datanodechown root:root -R /usr/local/hadoop # 更改目录所有者
hadoop name node -format # 格式化

启动

start-all.sh # 同时启动HDFS，yarn
stop-all.sh # 关闭
报错The authenticity of host '0.0.0.0 (0.0.0.0)' can't be established，
关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

查看

[root@localhost ~]# jps
2458 NodeManager
1524 DataNode
1445 NameNode
2746 Jps
2358 ResourceManager
1890 SecondaryNameNode启动成功。
HDFS： NameNode,SecondaryNameNode,DataNode
MapReduce2(Yarn): ResourceManager,NodeManager

打开Hadoop Resource-Manager Web界面

http://localhost:8088/

分布式集群配置：

master:

$ vim /etc/hostname

master

$ mkdir -p /usr/local/hadoop/hadoop_data/hdfs/namenode

$ vim /etc/hosts

192.168.31.110 master
192.168.31.111 data1
192.168.31.112 data2
192.168.31.113 data3

$ vim /usr/local/hadoop//etc/hadoop/slaves

data1
data2
data3

$ vim /usr/local/hadoop/etc/hadoop/core-site.xml

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
</configuration>

$ vim /usr/local/hadoop/etc/hadoop/yarn-site.xml

# ResourceManager主机与NodeManager的连接地址为8025

# ResourceManager 与 ApplicationMaster的连接地址为8030

# ResourceManager 与客户端的连接地址为8050

<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8025</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>master:8050</value>
</property></configuration>

$ vim /usr/local/hadoop/etc/hadoop/mapred-site.xml

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>master:54311</value>
</property>
</configuration>

$ vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml

data1 ~ data3:

$ vim /etc/hostname

data1

$ vim /etc/hosts

192.168.31.110 master
192.168.31.111 data1
192.168.31.112 data2
192.168.31.113 data3

$ mkdir -p /usr/local/hadoop/hadoop_data/hdfs/datanode

$ vim /usr/local/hadoop/etc/hadoop/core-site.xml

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
</configuration>

$ vim /usr/local/hadoop/etc/hadoop/yarn-site.xml

$ vim /usr/local/hadoop/etc/hadoop/mapred-site.xml

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>master:54311</value>
</property>
</configuration>

$ vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml

<configuration>
<property>
<name>dfs.replication</name>
<value>3</value>
</property> <property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop/hadoop_data/hdfs/datanode</value>
</property>
</configuration>

打开http://master:8088/

打开http://master:50070/