三 设置安装节点的 cloudera-manager yum信息
commonbase机器上：
vim /data/apache2/data/htdocs/cloudera-repos/cm6/6.3.1/redhat7/yum/cloudera-manager.repo
将地址修改为http://commonbase.example.com/cloudera-repos
[cloudera-manager]
name=Cloudera Manager 6.3.1
baseurl=http://commonbase.example.com/cloudera-repos/cm6/6.3.1/redhat7/yum/
gpgkey=http://commonbase.example.com/cloudera-repos/cm6/6.3.1/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md
[root@commonbase yum]# scp cloudera-manager.repo root@cdhmaster1:/etc/yum.repos.d/
cloudera-manager.repo 100% 235 0.6KB/s 00:00
[root@commonbase yum]# scp cloudera-manager.repo root@cdhmaster2:/etc/yum.repos.d/
cloudera-manager.repo 100% 235 0.2KB/s 00:01
[root@commonbase yum]# scp cloudera-manager.repo root@cdhnode1:/etc/yum.repos.d/
cloudera-manager.repo 100% 235 0.5KB/s 00:00
[root@commonbase yum]# scp cloudera-manager.repo root@cdhnode2:/etc/yum.repos.d/
cloudera-manager.repo 100% 235 1.3KB/s 00:00
[root@commonbase yum]# scp cloudera-manager.repo root@cdhnode3:/etc/yum.repos.d/
cloudera-manager.repo
四 开始安装
1 所有节点安装jdk
yum install oracle-j2sdk1.8
环境变量
export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH
[root@cdhmaster1 jdk1.8.0_181-cloudera]# scp /etc/profile root@cdhmaster2:/etc/profile
profile 100% 1970 198.0KB/s 00:00
[root@cdhmaster1 jdk1.8.0_181-cloudera]# scp /etc/profile root@cdhnode1:/etc/profile
profile 100% 1970 103.4KB/s 00:00
[root@cdhmaster1 jdk1.8.0_181-cloudera]# scp /etc/profile root@cdhnode2:/etc/profile
profile 100% 1970 3.9KB/s 00:00
[root@cdhmaster1 jdk1.8.0_181-cloudera]# scp /etc/profile root@cdhnode3:/etc/profile
profile 100% 1970 10.7KB/s 00:00
[root@cdhmaster1 jdk1.8.0_181-cloudera]# java -version
java version "1.8.0_181"
Java(TM) SE Runtime Environment (build 1.8.0_181-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.181-b13, mixed mode)
2 master上
[root@cdhmaster1 data]# yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
[root@cdhmaster2 data]# yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
3 node上
[root@cdhnode1 data]# yum install cloudera-manager-agent cloudera-manager-daemons
[root@cdhnode2 data]# yum install cloudera-manager-agent cloudera-manager-daemons
[root@cdhnode3 data]# yum install cloudera-manager-agent cloudera-manager-daemons
4 commonbase上
[root@commonbase parcels]# scp * root@cdhmaster1:/opt/cloudera/parcel-repo/
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel 100% 1986MB 24.8MB/s 01:20
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha 100% 41 30.1KB/s 00:00
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha1 100% 40 18.6KB/s 00:00
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha256 100% 64 40.6KB/s 00:00
manifest.json 100% 33KB 20.4MB/s 00:00
[root@commonbase parcels]# scp * root@cdhmaster2:/opt/cloudera/parcel-repo/
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel 100% 1986MB 23.9MB/s 01:23
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha 100% 41 8.8KB/s 00:00
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha1 100% 40 36.6KB/s 00:00
CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha256 100% 64 55.1KB/s 00:00
manifest.json
5 master上给刚才的parcels赋用户组
[root@cdhmaster1 parcel-repo]# chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
[root@cdhmaster2 data]# chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
6 node上修改agent配置文件
vim /etc/cloudera-scm-agent/config.ini
# Hostname of the CM server.
server_host=cdhmaster1.example.com
7 设置 Cloudera Manager 数据库
7.1 在mysql上执行
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
顺便把其他的数据库也创建CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Hive、Impala等元数据库CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'cdh'@'%' IDENTIFIED BY $mysql_password;
GRANT ALL privileges ON scm.* TO 'cdh'@'%';
GRANT ALL privileges ON amon.* TO 'cdh'@'%';
GRANT ALL privileges ON rman.* TO 'cdh'@'%';
GRANT ALL privileges ON hue.* TO 'cdh'@'%';
GRANT ALL privileges ON metastore.* TO 'cdh'@'%';
GRANT ALL privileges ON sentry.* TO 'cdh'@'%';
GRANT ALL privileges ON nav.* TO 'cdh'@'%';
GRANT ALL privileges ON navms.* TO 'cdh'@'%';
GRANT ALL privileges ON oozie.* TO 'cdh'@'%';
flush privileges;
7.2 初始化数据库
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh [options] <databaseType> <databaseName> <databaseUser> <password>
参数说明：
- options 指定操作，如果数据库不再本地请用
-h或--host指定mysql的host，不指定默认为localhost - databaseType 指定为mysql，也可以是其它类型的数据，例如：oracle等
- databaseName 指定为scm数据库，这里使用 scm库
- databaseUser 指定mysql用户名，这里使用 scm
- password 指定mysql其用户名的密码，这里使用scm
显然我这里是应该执行
./scm_prepare_database.sh -h commonbase.example.com -P 39106 -u root -p mysql scm cdh $mysql_password
Enter database password:
JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
Verifying that we can write to /etc/cloudera-scm-server
Sat Aug 29 12:01:44 CST 2020 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
Creating SCM configuration file in /etc/cloudera-scm-server
Executing: /usr/java/jdk1.8.0_181-cloudera/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cloudera/cm/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
Sat Aug 29 12:01:45 CST 2020 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
[ main] DbCommandExecutor INFO Successfully connected to database.
All done, your SCM database is configured correctly!
[root@cdhmaster1 schema]# scp /etc/cloudera-scm-server/db.properties root@cdhmaster2:/etc/cloudera-scm-server/db.properties
db.properties
8 安装CDH和其他服务
[root@cdhmaster1 schema]# systemctl start cloudera-scm-server
[root@cdhmaster1 schema]# systemctl enable cloudera-scm-server
[root@cdhmaster1 schema]# systemctl status cloudera-scm-server
● cloudera-scm-server.service - Cloudera CM Server Service
Loaded: loaded (/usr/lib/systemd/system/cloudera-scm-server.service; enabled; vendor preset: disabled)
Active: active (running) since 六 2020-08-29 12:04:58 CST; 10s ago
Main PID: 2771 (java)
CGroup: /system.slice/cloudera-scm-server.service
└─2771 /usr/java/jdk1.8.0_181-cloudera/bin/java -cp .:/usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connec...
8月 29 12:04:58 cdhmaster1.example.com cm-server[2771]: JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
8月 29 12:04:59 cdhmaster1.example.com cm-server[2771]: Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=2...in 8.0
8月 29 12:05:00 cdhmaster1.example.com cm-server[2771]: ERROR StatusLogger No log4j2 configuration file found. Using default con...gging.
8月 29 12:05:03 cdhmaster1.example.com cm-server[2771]: Sat Aug 29 12:05:03 CST 2020 WARN: Establishing SSL connection without s...applic
8月 29 12:05:03 cdhmaster1.example.com cm-server[2771]: Sat Aug 29 12:05:03 CST 2020 WARN: Establishing SSL connection without s...applic
8月 29 12:05:03 cdhmaster1.example.com cm-server[2771]: Sat Aug 29 12:05:03 CST 2020 WARN: Establishing SSL connection without s...applic
8月 29 12:05:03 cdhmaster1.example.com cm-server[2771]: Sat Aug 29 12:05:03 CST 2020 WARN: Establishing SSL connection without s...applic
8月 29 12:05:03 cdhmaster1.example.com cm-server[2771]: Sat Aug 29 12:05:03 CST 2020 WARN: Establishing SSL connection without s...applic
8月 29 12:05:06 cdhmaster1.example.com cm-server[2771]: 12:05:06.855 [main] ERROR org.hibernate.engine.jdbc.spi.SqlExceptionHelp... exist
8月 29 12:05:07 cdhmaster1.example.com cm-server[2771]: Sat Aug 29 12:05:07 CST 2020 WARN: Establishing SSL connection without s...applic
Hint: Some lines were ellipsized, use -l to show in full.
五 安装配置
打开浏览器 cdhmaster1.example.com:7180
登录Cloudera Manager Admin Console，默认凭证为
- Username: admin
- Password: admin
这一步选择安装的版本，不同版本支持的主要功能已列出，第一列为Cloudera免费的快速体验版；第二列为Cloudera为企业级试用版（免费试用60天）；第三列是功能和服务最全的Cloudera企业版，是需要认证且收费的。Cloudera Express 和 Cloudera Enterprise 中的可用功能的完整列表。
集群名称 Cluster Home
输入所有主机的域名 换行分割
选择版本后开始安装
因为要用spark 所以选择 Data Engineering
原则就是计算和node节点选所有机器，调度节点能选master1和master2就两台都选。
- 数据库设置：
| 服务 | 主机名称 | 数据库 | 用户名 | 密码 |
| :- | - | :- | - | - |
| Hive | commonbase.example.com:39106 | metastore | cdh | $mysql_password |
| Activity Monitor | commonbase.example.com:39106 | amon | cdh | $mysql_password |
| Oozie Server | commonbase.example.com:39106 | oozie | cdh | $mysql_password |
| Hue | commonbase.example.com:39106 | hue | cdh | $mysql_password |
一路过关斩将终于是完成了
最后部署成功后的状态 因为分配的配置低 问题还是挺多的 等运行一段时间稳定后就可以逐个解决了
宝剑锋从磨砺出,梅花香自苦寒来.