二十三 HDP搭建大数据环境

关闭Selinux
vi /etc/selinux/config
SELINUX=disabled关闭防火墙
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
firewall-cmd --state现在的epel源
[root@localhost ~]# yum install epel-release -y[root@localhost ~]# cat /etc/hostname
master
[root@localhost ~]# cat /etc/hostname
master[root@localhost ~]# yum install ntp -y
[root@localhost ~]# systemctl enable ntpd.service
[root@localhost ~]# systemctl start ntpd.service
[root@localhost ~]# ntpdate 0.centos.pool.ntp.org
[root@localhost ~]# yum install httpd -y
[root@localhost ~]# reboot安装httpd
[root@master ~]# yum install httpd -y
[root@master ~]# service httpd restart
[root@master ~]# chkconfig httpd on安装JDK
先卸载系统中自带的JDK
[root@master ~]# rpm -qa | grep openjdk
java-1.7.0-openjdk-1.7.0.171-2.6.13.2.el7.x86_64
java-1.8.0-openjdk-1.8.0.161-2.b14.el7.x86_64
java-1.7.0-openjdk-headless-1.7.0.171-2.6.13.2.el7.x86_64
java-1.8.0-openjdk-headless-1.8.0.161-2.b14.el7.x86_64
[root@master ~]# yum remove java-1.7.0-openjdk-headless-1.7.0.171-2.6.13.2.el7.x86_64
[root@master ~]# yum remove java-1.8.0-openjdk-headless-1.8.0.161-2.b14.el7.x86_64 -y
[root@master ~]# tar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/local/vi /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_171
export PATH=.:$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@master ~]# source /etc/profile
[root@master ~]# java -versionSSH无密钥登录
[root@master ~]# ssh-keygen
[root@master ~]# ssh-copy-id root@192.168.229.130
[root@master ~]# sudo sh -c "echo umask 0022 >> /etc/profile"设置系统运行参数
设置swappiness
[root@master ~]# echo 10 > /proc/sys/vm/swappiness
禁用 透明大页面压缩
[root@master ~]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@master ~]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
增加禁用透明大页面压缩的开机启动
[root@master ~]#vi /etc/rc.d/rc.local
if test -f /sys/kernel/mm/transparent_hugepage/enabled;
thenecho never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag;
thenecho never > /sys/kernel/mm/transparent_hugepage/defrag
fi
[root@master ~]#全部主机
修改配置文件/etc/sysctl.conf
kernel.shmmax= 1000000000
kernel.shmmni= 4096
kernel.shmall= 4000000000
kernel.sem= 250 512000 100 2048
kernel.sysrq= 1
kernel.core_uses_pid= 1
kernel.msgmnb= 65536
kernel.msgmax= 65536
kernel.msgmni= 2048
net.ipv4.tcp_syncookies= 0
net.ipv4.ip_forward= 0
net.ipv4.conf.default.accept_source_route= 0
net.ipv4.tcp_tw_recycle= 1
net.ipv4.tcp_max_syn_backlog= 200000
net.ipv4.conf.all.arp_filter= 1
net.ipv4.ip_local_port_range= 1281 65535
net.core.netdev_max_backlog= 200000
#java程序多时，设成2会导致很多服务不能去启动！
#vm.overcommit_memory= 2
vm.overcommit_memory=0
fs.nr_open= 3000000
kernel.threads-max= 798720
kernel.pid_max= 798720
#increase network
net.core.rmem_max=2097152
net.core.wmem_max=2097152[root@master ~]# sysctl -p修改/etc/security/limits.conf设置打开文件数限制
vim /etc/security/limits.conf
* soft nofile 2900000
* hard nofile 2900000
* soft nproc 131072
* hard nproc 131072重新登录以使配置生效，或者使用命令临时设置
ulimit -n 10240解压HDP3.0软件包到/var/www/html目录下
[root@master html]# tar -zxvf /root/ambari-2.7.1.0-centos7.tar.gz -C .
[root@master html]# tar -zxvf /root/HDP-3.0.1.0-centos7-rpm.tar.gz -C ./hdp/
[root@master html]# tar -zxvf /root/HDP-GPL-3.0.1.0-centos7-gpl.tar.gz -C ./hdp/
[root@master html]# tar -zxvf /root/HDP-UTILS-1.1.0.22-centos7.tar.gz -C ./hdp//etc/yum.repos.d配置文件
[root@master yum.repos.d]# cat ambari.repo
#VERSION_NUMBER=2.7.1.0-169
[ambari-2.7.1.0]
#json.url = http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json
name=ambari Version - ambari-2.7.1.0
baseurl=http://192.168.229.130/ambari/centos7/2.7.1.0-169/
gpgcheck=1
gpgkey=http://192.168.229.130/ambari/centos7/2.7.1.0-169/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1[root@master yum.repos.d]# cat hdp.repo
#VERSION_NUMBER=3.0.1.0-187
[HDP-3.0.1.0]
name=HDP Version - HDP-3.0.1.0
baseurl=http://192.168.229.130/hdp/HDP/centos7/3.0.1.0-187
gpgcheck=1
gpgkey=http://192.168.229.130/hdp/HDP/centos7/3.0.1.0-187/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1[HDP-UTILS-1.1.0.22]
name=HDP-UTILS Version - HDP-UTILS-1.1.0.22
baseurl=http://192.168.229.130/hdp/HDP-UTILS/centos7/1.1.0.22
gpgcheck=1
gpgkey=http://192.168.229.130/hdp/HDP/centos7/3.0.1.0-187/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1#VERSION_NUMBER=3.0.1.0-187
[HDP-GPL-3.0.1.0]
name=HDP-GPL Version - HDP-GPL-3.0.1.0
baseurl=http://192.168.229.130/hdp/HDP-GPL/centos7/3.0.1.0-187
gpgcheck=1
gpgkey=http://192.168.229.130/hdp/HDP-GPL/centos7/3.0.1.0-187/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1[root@master yum.repos.d]#
使用刚才配置好的本地仓库，直接yum命令安装
[root@master ~]# yum install ambari-server -y安装mysql第一次收集软件包
卸载mariadb
[root@master ~]# rpm -qa | grep mariadb
mariadb-libs-5.5.56-2.el7.x86_64
[root@master ~]# yum remove mariadb-libs-5.5.56-2.el7.x86_64wget http://dev.mysql.com/get/mysql57-community-release-el7-7.noarch.rpm
rpm -ivh ./mysql57-community-release-el7-7.noarch.rpm
安装连接器yum install mysql-connector-java -y
[root@master ~]# yum install mysql-server -y
[root@master ~]# chkconfig mysqld on
[root@kolla ~]# yum install mysql-connector-java -y启动mysql,设置开机启动
[root@master ~]# service mysqld start
[root@master ~]# vi /etc/rc.local
#添加service mysqld start查看mysql密码
[root@master ~]# grep "temporary password" /var/log/mysqld.log
2019-01-05T07:07:14.924796Z 1 [Note] A temporary password is generated for root@localhost: ,hbjy*P7aB<O
[root@master ~]# mysql -uroot -p
Enter password:,hbjy*P7aB<O
mysql> set global validate_password_policy=0;
mysql> set global validate_password_length=3;  #设置密码长度最小为6个字符
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';
设置mysql远程链接
mysql> grant all privileges on *.* to 'root'@'%' identified by '123456';登录mysql ,执行下面的语句，创建ambari数据库，为了在ambari-server安装后启动，否则报错
mysql> create database ambari;
mysql> use ambari;
mysql> CREATE USER 'ambari'@'%' IDENTIFIED BY '123456';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%';
mysql> CREATE USER 'ambari'@'localhost' IDENTIFIED BY '123456';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';
mysql> CREATE USER 'ambari'@'master' IDENTIFIED BY '123456';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'master';
mysql> flush privileges;
mysql> source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql配置 ambari server
[root@master ~]# ambari-server setup
Customize user account for ambari-server daemon [y/n] (n)? n
Checking JDK...
[2] Custom JDK
==============================================================================
Enter choice (1): 2
Path to JAVA_HOME: /usr/local/jdk1.8.0_171
Enable Ambari Server to download and install GPL Licensed LZO packages [y/n] (n)? y
Enter advanced database configuration [y/n] (n)? y
==============================================================================
Choose one of the following options:
[1] - PostgreSQL (Embedded)
[2] - Oracle
[3] - MySQL / MariaDB
[4] - PostgreSQL
[5] - Microsoft SQL Server (Tech Preview)
[6] - SQL Anywhere
[7] - BDB
==============================================================================
Enter choice (1): 3
Hostname (localhost): master
Port (3306):
Database name (ambari): ambari
Username (ambari): root                        #采用root账号超级权限
Enter Database Password (bigdata): 123456      #采用root账号超级权限
Re-enter password: 123456                      #采用root账号超级权限
Should ambari use existing default jdbc /usr/share/java/mysql-connector-java.jar [y/n] (y)? y
Proceed with configuring remote database connection properties [y/n] (y)? yAdjusting ambari - server permissions and ownership...
Ambari Server 'setup' completed successfully.
[root@master ~]#命令行方式设置mysql数据库链接库，用于oozie和range链接mysql时使用（如果上述操作不起作用，则需要采用下面操作）
[root@master ~]# ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar
Using python  /usr/bin/python
Setup ambari-server
Copying /usr/share/java/mysql-connector-java.jar to /var/lib/ambari-server/resources/mysql-connector-java.jar
If you are updating existing jdbc driver jar for mysql with mysql-connector-java.jar. Please remove the old driver jar, from all hosts. Restarting services that need the driver, will automatically copy the new jar to the hosts.
JDBC driver was successfully initialized.
Ambari Server 'setup' completed successfully.
[root@master ~]#配置ambari-server自启动，启动ambari-server
[root@master ~]# systemctl enable ambari-server
[root@master ~]# systemctl start ambari-server浏览器访问ambari-server服务
http://192.168.229.130:8080     账号admin密码admin
登录之后不要动其它的，先安装ambari-agent，否则后续会出现问题
所有主机节点，安装ambari-agent,并配置自动启动
[root@master ~]# yum install ambari-agent -y
[root@master ~]# systemctl enable ambari-agent
[root@master ~]# systemctl restart ambari-agent && systemctl status ambari-agent如果出现问题，应该是hive和oozie数据库没有创建，则继续安装
创建oozie库
mysql> create database oozie  DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

参考文档

https://www.cnblogs.com/dajianshi/p/9473650.html

二十三 HDP搭建大数据环境相关推荐

从搭建大数据环境说起，到执行WordCount所遇到的坑
从搭建大数据环境说起,到执行WordCount所遇到的坑文章目录从搭建大数据环境说起,到执行WordCount所遇到的坑背景说明基于`docker compose`的大数据环境搭建 docke ...
5分钟搭建大数据学习环境
想学大数据,先搭个环境再说,要准备多少台服务器,每台服务器的基础环境设置,各个服务器的网络互联,真是很麻烦的一件事第一座山先把想入门的学习者挡在门外,别放弃,这里又一个即全面又快捷的搭建大数据环境的 ...
在阿里云中搭建大数据实验环境
云计算是和自来水一样的商业模式.有了自来水,我们可以不用自己挖井,就可以快速.便捷.廉价地获得水资源.而有了云计算,我们就可以不用自己建机房.买设备.维护系统,就可以快速.便捷.廉价地获得IT资源(C ...
[转]大数据环境搭建步骤详解（Hadoop，Hive，Zookeeper，Kafka，Flume，Hbase，Spark等安装与配置）
大数据环境安装和配置(Hadoop2.7.7,Hive2.3.4,Zookeeper3.4.10,Kafka2.1.0,Flume1.8.0,Hbase2.1.1,Spark2.4.0等) 系统说明 ...
使用O2OA二次开发搭建企业办公平台（十三）流程开发篇：报销审批流程表单开发...
本博客为O2OA系列教程.O2OA使用手册,教程目录和各章节天梯将在连载完后更新. 使用O2OA二次开发搭建企业办公平台(一)平台部署篇:平台下载和部署使用O2OA二次开发搭建企业办公平台(二)平台 ...
使用O2OA二次开发搭建企业办公平台（十三）流程开发篇：报销审批流程表单开发
本博客为O2OA系列教程.O2OA使用手册,教程目录和各章节天梯将在连载完后更新. 使用O2OA二次开发搭建企业办公平台(一)平台部署篇:平台下载和部署使用O2OA二次开发搭建企业办公平台(二)平台 ...
【阿尼亚喜欢BigData】“红亚杯”大数据环境搭建与数据采集技能线上专题赛——满分解析⑤
大家好,喜欢Bigdata的阿尼亚来了!希望大家会喜欢阿尼亚的文章!!哇酷哇酷!!! 本次为师傅们带来的是"红亚杯"大数据环境搭建与数据采集技能线上专题赛--满分解析系列的第⑤期, ...
【阿尼亚喜欢BigData】“红亚杯”大数据环境搭建与数据采集技能线上专题赛——满分解析③
大家好,喜欢Bigdata的阿尼亚来了!希望大家会喜欢阿尼亚的文章!!哇酷哇酷!!! 本次为师傅们带来的是"红亚杯"大数据环境搭建与数据采集技能线上专题赛--满分解析系列的第③期, ...
【阿尼亚喜欢BigData】“红亚杯”大数据环境搭建与数据采集技能线上专题赛——满分解析②
大家好,喜欢Bigdata的阿尼亚来了!希望大家会喜欢阿尼亚的文章!!哇酷哇酷!!! 本次为师傅们带来的是"红亚杯"大数据环境搭建与数据采集技能线上专题赛--满分解析系列的第②期, ...

二十三 HDP搭建大数据环境

二十三 HDP搭建大数据环境相关推荐

最新文章

热门文章