一、集群搭建&MySQL部署
1.首先使用青云搭建了大数据集群，集群有三台机器，分别名为hadoop001,hadoop002,hadoop003
2.在hadoop001机器上部署MySQL
二、环境准备
***步骤3~8集群的每个机器都要操作一次
[root@hadoop001 ~]# service iptables stop
[root@hadoop001 ~]# service iptables status //验证
[root@hadoop001 ~]# chkconfig iptables off
[root@hadoop001 ~]# chkconfig --list | grep iptables //验证
4.检查python版本(系统默认的python版本不要修改，如果生产上跑Python服务需要更高版本，就另外装一个)
[root@hadoop001 ~]# python --version
Python 2.6.6
5.配置NTP网络时间协议，同步时区和时间
[root@hadoop001 ~]# grep ZONE /etc/sysconfig/clock
ZONE="Asia/Shanghai"
[root@hadoop001 ~]# ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
[root@hadoop001 ~]# yum install -y ntpdate
[root@hadoop001 ~]# chkconfig ntpd on
//上面的命令要在三台机器都执行，下面的只在hadoop001执行
[root@hadoop001 ~]# cp /etc/ntp.conf /etc/ntp.conf.bak
[root@hadoop001 ~]# cp /etc/sysconfig/ntpd /etc/sysconfig/ntpd.bak
[root@hadoop001 ~]# echo "restrict 192.168.137.0 mask 255.255.255.0 nomodify notrap" >> /etc/ntp.conf
[root@hadoop001 ~]# echo "SYNC_HWCLOCK=yes" >> /etc/sysconfig/ntpd
//上面的命令只在hadoop001执行，下面的要在三台机器都执行
[root@hadoop001 ~]# service ntpd restart
[root@hadoop001 ~]# crontab -e
*/30 * * * * /usr/sbin/ntpdate 192.168.137.2
no crontab for root - using an empty one
crontab: installing new crontab
6.关闭大页面
[root@hadoop001 ~]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@hadoop001 ~]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
[root@hadoop001 ~]# echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag'>> /etc/rc.local
[root@hadoop001 ~]# echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled'>> /etc/rc.local
7.swap 物理磁盘空间可以内存使用
[root@hadoop001 ~]# echo 'vm.swappiness = 10' >> /etc/sysctl.conf
[root@hadoop001 ~]# sysctl -p
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
vm.swappiness = 10
swap的取值范围是0~100，0不代表禁用，但是惰性最高；100代表积极性最高
对集群计算的实时性要求高的，swap设置为0，能不使用磁盘当内存就不使用，允许job挂 迅速的加内存或调大参数然后重启job；
对集群计算的实时性要求不高的，swap一般设置为10或30，不允许job挂，让job慢慢的运行
8.安装http和启动http服务
[root@hadoop001 ~]# yum install -y httpd
[root@hadoop001 ~]# rpm -qa|grep httpd
httpd-2.2.15-69.el6.centos.x86_64
httpd-tools-2.2.15-69.el6.centos.x86_64
[root@hadoop001 ~]# chkconfig --list|grep httpd
httpd 0:off 1:off 2:off 3:off 4:off 5:off 6:off
[root@hadoop001 ~]# chkconfig httpd on
[root@hadoop001 ~]# chkconfig --list|grep httpd
httpd 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@hadoop001 ~]# service httpd start
Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 192.168.137.2 for ServerName
[ OK ]
三、配置离线数据源
****步骤9~11只在server一台机器上创建即可
9.创建 parcels文件
[root@hadoop001 ~]# cd /var/www/html
[root@hadoop001 html]# mkdir parcels
[root@hadoop001 html]# cd parcels
[root@hadoop001 parcels]# rz (上传CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel、CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha1、manifest.json三个文件，然后将CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha1重命名)
[root@hadoop001 parcels]# ll
total 1581984
-rw-r--r-- 1 root root 1619866604 Oct 12 20:19 CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel
-rw-r--r-- 1 root root 41 Oct 12 20:16 CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha1
-rw-r--r-- 1 root root 72612 Oct 12 20:17 manifest.json
[root@hadoop001 parcels]# mv CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha1 CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha
[root@hadoop001 parcels]# ll
total 1581984
-rw-r--r-- 1 root root 1619866604 Oct 12 20:19 CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel
-rw-r--r-- 1 root root 41 Oct 12 20:16 CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha
-rw-r--r-- 1 root root 72612 Oct 12 20:17 manifest.json
10.校验parcels文件下载未损坏（生产上因为没校验吃过亏）
[root@hadoop001 parcels]# sha1sum CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel
c37d5191214db7b65b703e7b51ff2853f2fb9dd9 CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel
[root@hadoop001 parcels]# cat CDH-5.12.0-1.cdh5.12.0.p0.29-el6.parcel.sha
c37d5191214db7b65b703e7b51ff2853f2fb9dd9
11.下载、上传、解压cm5.12.0-centos6.tar.gz，然后移动到和官网一样的目录路径cm5/redhat/6/x86_64/下
[root@hadoop001 parcels]# mkdir -p /opt/rpminstall
[root@hadoop001 parcels]# cd /opt/rpminstall
[root@hadoop001 rpminstall]# ll
total 955988
-rw-r--r-- 1 root root 978923802 Oct 12 20:59 cm5.12.0-centos6.tar.gz
[root@hadoop001 rpminstall]# tar -zxf cm5.12.0-centos6.tar.gz -C /var/www/html/
[root@hadoop001 rpminstall]# cd /var/www/html/
[root@hadoop001 html]# ll
total 8
drwxrwxr-x 3 1106 592 4096 Jul 7 2017 cm
drwxr-xr-x 2 root root 4096 Oct 12 20:49 parcels
[root@hadoop001 html]# mkdir -p cm5/redhat/6/x86_64/
[root@hadoop001 html]# mv cm cm5/redhat/6/x86_64/
***步骤12每个机器都要配置cloudera-manager.repo文件
12.配置本地的yum源,cdh集群在安装时会就从本地down包,不会从官网了
[root@hadoop001 html]# vi /etc/yum.repos.d/cloudera-manager.repo
[cloudera-manager]
name = Cloudera Manager, Version 5.12.0
baseurl = http://192.168.137.2/cm5/redhat/6/x86_64/cm/5/
gpgcheck = 0
浏览器查看下面两个网址是否出来,假如有，就配置成功
http://192.168.137.2/parcels/
http://192.168.137.2/cm5/redhat/6/x86_64/cm/5/
(192.168.137.2要替换为外网IP)
四、使用RPM包安装并启动CM服务
13.在cm实例中安装服务器rpm包
[root@hadoop001 ~]# cd /var/www/html/cm5/redhat/6/x86_64/cm/5/RPMS/x86_64
[root@hadoop001 x86_64]# ll
total 958396
-rw-rw-r-- 1 1106 592 9674128 Jul 7 2017 cloudera-manager-agent-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
-rw-rw-r-- 1 1106 592 726564488 Jul 7 2017 cloudera-manager-daemons-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
-rw-rw-r-- 1 1106 592 8704 Jul 7 2017 cloudera-manager-server-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
-rw-rw-r-- 1 1106 592 10608 Jul 7 2017 cloudera-manager-server-db-2-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
-rw-rw-r-- 1 1106 592 31880440 Jul 7 2017 enterprise-debuginfo-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
-rw-rw-r-- 1 1106 592 71204325 Jul 7 2017 jdk-6u31-linux-amd64.rpm
-rw-rw-r-- 1 1106 592 142039186 Jul 7 2017 oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm
[root@hadoop001 x86_64]# yum install -y cloudera-manager-daemons-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
[root@hadoop001 x86_64]# yum install -y cloudera-manager-server-5.12.0-1.cm5120.p0.120.el6.x86_64.rpm
14.配置mysql-connector-java.jar
[root@hadoop001 x86_64]# cd /usr/share/java
[root@hadoop001 x86_64]# rz
[root@hadoop001 java]# ll
total 968
-rw-r--r-- 1 root root 989495 Sep 27 07:05 mysql-connector-java.jar
***注意一定要把版本号去掉，名字改成'mysql-connector-java.jar'
15.MySQL配置cmf 用户 and 数据库
mysql> create database cmf DEFAULT CHARACTER SET utf8;
Query OK, 1 row affected (0.00 sec)
mysql> grant all on cmf.* TO 'cmf'@'%' IDENTIFIED BY 'cmf_password';
Query OK, 0 rows affected (0.00 sec)
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
16.设置cloudera-scm-server 连接到 MySQL
[root@hadoop001 java]# cd /etc/cloudera-scm-server/
[root@hadoop001 cloudera-scm-server]# vi db.properties
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# This file describes the database connection.
#
# The database type
# Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
com.cloudera.cmf.db.type=mysql
# The database host
# If a non standard port is needed, use 'hostname:port'
com.cloudera.cmf.db.host=192.168.137.2:3306
# The database name
com.cloudera.cmf.db.name=cmf
# The database user
com.cloudera.cmf.db.user=cmf
# The database user's password
com.cloudera.cmf.db.password=cmf_password
# The db setup type
# By default, it is set to INIT
# If scm-server uses Embedded DB then it is set to EMBEDDED
# If scm-server uses External DB then it is set to EXTERNAL
com.cloudera.cmf.db.setupType=EXTERNAL
17.启动cm server
[root@hadoop001 cloudera-scm-server]# service cloudera-scm-server start
Starting cloudera-scm-server: [ OK ]
//可以查看下方日志来实时观察cm server启动是否有问题
[root@hadoop001 cloudera-scm-server]# cd /var/log/cloudera-scm-server/
[root@hadoop001 cloudera-scm-server]# tail -f cloudera-scm-server.log
2018-10-13 13:13:57,831 INFO WebServerImpl:org.mortbay.log: Started SelectChannelConnector@0.0.0.0:7180
2018-10-13 13:13:57,831 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
****此处遇见一个坑，因为用的云主机，一定要设置内外网端口转发和放开防火墙7180端口，不然死活都打不开ip:7180
18.打开web界面，安装CM server 和 agent
19.集群一系列设置
//cm监控需要一个数据库配置；一个cm需要一个数据库，这里我们新建一个amon数据库
mysql> create database amon DEFAULT CHARACTER SET utf8;
Query OK, 1 row affected (0.01 sec)
mysql> grant all on amon.* TO 'amon'@'%' IDENTIFIED BY 'amon_password';
Query OK, 0 rows affected (0.00 sec)
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
20.kafka部署
https://www.cloudera.com/documentation/kafka/latest/topics/kafka_installing.html#concept_m2t_d45_4r
[root@hadoop001 ~]# cd /var/www/html
[root@hadoop001 html]# mkdir kafka_parcels
[root@hadoop001 html]# cd kafka_parcels
[root@hadoop001 html]# rz
[root@hadoop001 kafka_parcels]# ll
total 66536
-rw-r--r-- 1 root root 68116503 Oct 13 17:05 KAFKA-2.1.1-1.2.1.1.p0.18-el6.parcel
-rw-r--r-- 1 root root 41 Oct 13 17:04 KAFKA-2.1.1-1.2.1.1.p0.18-el6.parcel.sha1
-rw-r--r-- 1 root root 5252 Feb 6 2018 manifest.json
[root@hadoop001 kafka_parcels]# mv KAFKA-2.1.1-1.2.1.1.p0.18-el6.parcel.sha1 KAFKA-2.1.1-1.2.1.1.p0.18-el6.parcel.sha
[root@hadoop001 kafka_parcels]# ll
total 66536
-rw-r--r-- 1 root root 68116503 Oct 13 17:05 KAFKA-2.1.1-1.2.1.1.p0.18-el6.parcel
-rw-r--r-- 1 root root 41 Oct 13 17:04 KAFKA-2.1.1-1.2.1.1.p0.18-el6.parcel.sha
-rw-r--r-- 1 root root 5252 Feb 6 2018 manifest.json
如果安装过程中出了问题，在机器上[root@hadoop001 ~]# cd /var/log/目录下找到的都是role log（角色日志）很多情况下找不到问题出在哪里，这种情况下应该去看stdout，stderr日志，通过web界面找
五、如何停止集群，启动集群
[root@hadoop001 ~]# service cloudera-scm-agent stop
Stopping cloudera-scm-agent: [ OK ]
//上面的命令要在三台机器都执行，下面的只在hadoop001执行
[root@hadoop001 ~]# service cloudera-scm-server stop
Stopping cloudera-scm-server: [ OK ]
[root@hadoop001 ~]# service mysql stop
Shutting down MySQL.. [ OK ]
（启动时反过来就行了，注意mysql是部署在mysqladmin用户下，service mysql start要在mysqladmin用户）
[root@hadoop001 ~]# su - mysqladmin
[mysqladmin@hadoop001 ~]$ service mysql start
Starting MySQL [ OK ]
[root@hadoop001 ~]# service cloudera-scm-server start
//上面的命令只在hadoop001执行，下面的要在三台机器都执行
[root@hadoop001 ~]# service cloudera-scm-agent start
Starting cloudera-scm-agent: [ OK ]
