一. 部署架构说明
openshift 4.x 开始,部署方式不再依赖于3.x的ansible 部署方式,4.x开始部署方式有较大的改变,本次部署使用的是裸金属的部署方式,使用dhcp无人值守安装部署.
# 部署流程说明
# 1.) 部署机,部署dns,dhcp,tftp,haproxy,
# 2.) 部署机,通过pxe的方式,部署出引导机,
# 3.) 引导机器部署完成之后,通过引导机,部署出openshift4.3集群
二. 部署环境规划
三. 部署离线环境.(安装部署机)
a. ) 配置内部镜像仓库
#安装podman或docker,http-tools,http,
#yum -y install podman httpd httpd-tools
# 创建内部镜像仓库证书目录,数据目录,认证目录
# mkdir -p /opt/registry/{auth,certs,data}
# 生成tls 证书文件
# cd /opt/registry/certs
# 请注意证书的域名.
# openssl req -subj '/CN=registry.example.com/O=My Company Name LTD./C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout domain.key -out domain.crt
# 使用htpasswd 生成用户名密码文件
# htpasswd -bBc /opt/registry/auth/htpasswd test test
# 确保你的5000端口已在防火墙中放行
# 运行内部仓库镜像
# podman run --name poc-registry -p 5000:5000 \
-v /opt/registry/data:/var/lib/registry:z \
-v /opt/registry/auth:/auth:z \
-e "REGISTRY_AUTH=htpasswd" \
-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" \
-e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" \
-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-v /opt/registry/certs:/certs:z \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
-e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
docker.io/library/registry:2
# 使用下面的命令验证内部仓库是否可以正常访问,这里应该返回一个空值,因为仓库中并没有镜像
# curl -u test:test -k https://registry.example.com:5000/v2/_catalog
# 如仓库无法访问,您应该检查
# 1.) registry 容器是否正常启动
# 2.) registry.example.com 域名是否可以正常解析
# 3.) htpasswd 文件中的用户名密码是否正确
# 4.) 防火墙5000端口是否开放
# 5.) 证书域名是否正确
b .) 获取离线镜像
# 下载oc客户端,您应该下载latest版本来部署你的openshift集群
# 请从当前页面下载 https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.3/
#从https://cloud.redhat.com/openshift/install/pre-release获取pull-secret.json 秘钥
# 将本地镜像仓库中的验证信息使用base64 进行加密
# echo -n 'test:test' | base64 -w0
# 修改pull-secret.json 文件,将本地仓库验证信息加入
# 设置环境变量
export OCP_RELEASE="4.3.0-x86_64"
# 本地镜像仓库域名端口
export LOCAL_REGISTRY='registry.example.com:5000'
# 本地镜像仓库存储库名称
export LOCAL_REPOSITORY='ocp4/openshift4'
# ocp发布的版本,一般不需要更改,也可以从https://mirror.openshift.com/pub/openshift-v4/clients/ocp-dev-preview/latest/release.txt获得
export PRODUCT_REPO='openshift-release-dev'
# 拉取ocp镜像的秘钥
export LOCAL_SECRET_JSON="./pull-secret-2.json"
# release版本名称,
export RELEASE_NAME="ocp-release"
# 将自签名的证书复制到默认信任证书路径
# cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/
#
update-ca-trust extract
#使用 oc 命令拉取离线镜像,并推送到自己的内部仓库
oc adm -a ${LOCAL_SECRET_JSON} releasemirror \
--from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE} \
--to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} \
--to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}
# 1.) 如pull镜像失败,大部分是超时引起的,需要从新执行脚本拉取镜像
# 2.) 每次拉取都会从头开始
# 离线部署的openshift-install 安装文件需要通过离线镜像创建,创建时需要连网# 载入前面创建镜像仓库的环境变量
# oc adm -a ${LOCAL_SECRET_JSON} release extract --command=openshift-install "${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}"# cp openshift-install /usr/local/bin/
c.) 部署dnsmasq ,用于提供dns,dhcp ,tftp,服务
# 安装dnsmasq
# yum installdnsmasq tftp-server ipxe-bootimgs -y
# systemctlenable dnsmasq && systemctl start dnsmasq
# 配置dnsmasq ,可以参考我的配置文件配置
# ############################
dhcp-range=192.168.1.173,192.168.1.180,255.255.248.0
enable-tftp
tftp-root=/var/lib/tftpboot
dhcp-match=set:bios,option:client-arch,0
dhcp-boot=tag:bios,undionly.kpxe
dhcp-match=set:efi32,option:client-arch,6
dhcp-boot=tag:efi32,ipxe.efi
dhcp-match=set:efibc,option:client-arch,7
dhcp-boot=tag:efibc,ipxe.efi
dhcp-match=set:efi64,option:client-arch,9
dhcp-boot=tag:efi64,ipxe.efi
dhcp-userclass=set:ipxe,iPXE
dhcp-option=option:router,192.168.1.1
dhcp-option=option:dns-server,192.168.1.171
dhcp-boot=tag:ipxe,http://bastion.ocp67.example.com:8080/boot.ipxe
address=/bastion.ocp67.example.com/192.168.1.171
address=/api.ocp67.example.com/192.168.1.171
address=/apps.ocp67.example.com/192.168.1.176
address=/api-int.ocp67.example.com/192.168.1.171
address=/master-0.ocp67.example.com/192.168.1.173
address=/etcd-0.ocp67.example.com/192.168.1.173
address=/master-1.ocp67.example.com/192.168.1.174
address=/etcd-1.ocp67.example.com/192.168.1.174
address=/master-2.ocp67.example.com/192.168.1.175
address=/etcd-2.ocp67.example.com/192.168.1.175
address=/node-0.ocp67.example.com/192.168.1.176
address=/node-1.ocp67.example.com/192.168.1.177
address=/node-2.ocp67.example.com/192.168.1.179
address=/bootstrap-0.ocp67.example.com/192.168.1.178
address=/registry.example.com/192.168.1.172
srv-host=_etcd-server-ssl._tcp.ocp67.example.com,etcd-0.ocp67.example.com,2380,10
srv-host=_etcd-server-ssl._tcp.ocp67.example.com,etcd-1.ocp67.example.com,2380,10
srv-host=_etcd-server-ssl._tcp.ocp67.example.com,etcd-2.ocp67.example.com,2380,10
dhcp-host=00:50:56:a2:39:71,master-0.ocp67.example.com,192.168.1.173
dhcp-host=00:50:56:a2:1b:bc,master-1.ocp67.example.com,192.168.1.174
dhcp-host=00:50:56:a2:1d:26,master-2.ocp67.example.com,192.168.1.175
dhcp-host=00:50:56:a2:25:f2,192.168.1.176
dhcp-host=00:50:56:a2:3f:4b,192.168.1.177
dhcp-host=00:50:56:a2:63:85,192.168.1.179
dhcp-host=00:50:56:a2:77:f0,192.168.1.178
log-queries
log-dhcp
####################################.
# 配置tftp server
# mkdir -p/var/lib/tftpboot
# ln -s /usr/share/ipxe/undionly.kpxe /var/lib/tftpboot
#开启防火墙策略,确保你的dns,dhcp,可以被正确访问到
# 启动dnsmasq
# systemctl enable dnsmasq && systemctl restart
dnsmasq
d.) 配置haproxy ,为api server,bootstarp ,提供负载均衡,
# 部署haproxy 负载均衡
# 该haproxy 主要是为master ,infra 节点提供负载均衡,可以使用其他负载均衡替代.
# 配置yum源,安装haproxy .
# yum install haproxy -y
# 配置haproxy 配置文件,增加master负载,infra负载
# ##############################
frontendk8s-int
bind *:22623
mode tcp
default_backend app1
backend app1
balance source
mode tcp
server bootstarp 192.168.1.178:22623 check
server master-0 192.168.1.173:22623 check
server master-1 192.168.1.174:22623 check
server master-2 192.168.1.175:22623 check
frontend k8s
bind *:6443
mode tcp
default_backend app2
backend app2
balance source
mode tcp
server bootstarp 192.168.1.178:6443 check
server master-0 192.168.1.173:6443 check
server master-1 192.168.1.174:6443 check
server master-2 192.168.1.175:6443 check
frontendinfra-0
bind *:443
mode tcp
default_backend app3
backend app3
balance source
mode tcp
server infra-0 192.168.1.177:443 check
frontendinfra-http
bind *:80
mode tcp
default_backend app4
backend app4
balance source
mode tcp
server infra-0 192.168.1.177:80 check
####################################
# 启动haproxy ,确保6443,22623,80,443 端口处于haproxy监听状态
# systemctlenable haproxy && systemctl start haproxy
e.) 配置matchbox,该服务主要是用作pxe安装时,分配ign部署配置文件
# 下载matchbox,可以到github的release页面下载, https://github.com/poseidon/matchbox/releases
# 解压 matchbox-v0.8.3-linux-amd64.tar.gz 安装包
# tar xf matchbox-v0.8.3-linux-amd64.tar.gz
# cdmatchbox-v0.8.3-linux-amd64/
# cp matchbox/usr/local/bin
# cp contrib/systemd/matchbox-local.service/etc/systemd/system/matchbox.service
# mkdir -p/var/lib/matchbox/{assets,groups,ignition,profiles}
# cd /var/lib/matchbox/assets
# 下载rhcos 安装文件,你可以从这里下载: https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/pre-release/latest/
# 需要下载:
# 1.) rhcos-43.81.201912030353.0-installer-initramfs.x86_64.img
# 2.) rhcos-43.81.201912030353.0-installer-kernel-x86_64
# 3.) rhcos-43.81.201912030353.0-metal.x86_64.raw.gz
# 配置pxe自动部署配置文件
# /var/lib/matchbox/profiles/bootstrap.json
##################################
{
"id": "bootstrap",
"name": "OCP 4 – Bootstrap",
"ignition_id": "bootstrap.ign",
"boot": {
"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",
"initrd": [
"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"
],
"args": [
"ip=dhcp",
"rd.neednet=1",
"console=tty0",
"console=ttyS0",
"coreos.inst=yes",
"coreos.inst.install_dev=sda",
"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",
"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"
]
}
}
# /var/lib/matchbox/profiles/master.json
{
"id": "master",
"name": "OCP 4 – Master",
"ignition_id": "master.ign",
"boot": {
"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",
"initrd": [
"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"
],
"args": [
"ip=dhcp",
"rd.neednet=1",
"console=tty0",
"console=ttyS0",
"coreos.inst=yes",
"coreos.inst.install_dev=sda",
"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",
"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"
]
}
}
##############
# /var/lib/matchbox/profiles/infnod.json
{
"id": "infnod",
"name": "OCP 4 – Infrastructure Node",
"ignition_id": "worker.ign",
"boot": {
"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",
"initrd": [
"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"
],
"args": [
"ip=dhcp",
"rd.neednet=1",
"console=tty0",
"console=ttyS0",
"coreos.inst=yes",
"coreos.inst.install_dev=sda",
"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",
"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"
]
}
}
# /var/lib/matchbox/profiles/cptnod.json
{
"id": "cptnod",
"name": "OCP 4 – Compute Node",
"ignition_id": "worker.ign",
"boot": {
"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",
"initrd": [
"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"
],
"args": [
"ip=dhcp",
"rd.neednet=1",
"console=tty0",
"console=ttyS0",
"coreos.inst=yes",
"coreos.inst.install_dev=sda",
"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",
"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"
]
}
}
# /var/lib/matchbox/groups/bootstrap-0.json
{
"id": "bootstrap-0",
"name": "OCP 4 – Bootstrap server",
"profile": "bootstrap",
"selector": {
"mac": "00:50:56:a2:77:f0"
}
}
# /var/lib/matchbox/groups/master-{0-2}.json
{
"id": "master-1",
"name": "OCP 4 – Master 2",
"profile": "master",
"selector": {
"mac": "00:50:56:a2:1b:bc"
}
}
{
"id": "master-2",
"name": "OCP 4 – Master 3",
"profile": "master",
"selector": {
"mac": "00:50:56:a2:1d:26"
}
}
{
"id": "master-0",
"name": "OCP 4 – Master 1",
"profile": "master",
"selector": {
"mac": "00:50:56:a2:39:71"
}
}
#/var/lib/matchbox/groups/infnod-0.json
{
"id": "infnod-0",
"name": "OCP 4 – Infrastructure Node #1",
"profile": "infnod",
"selector": {
"mac": "00:50:56:a2:25:f2"
}
}
# /var/lib/matchbox/groups/cptnod-{0-1}.json
{
"id": "cptnod-0",
"name": "OCP 4 – Compute node #1",
"profile": "cptnod",
"selector": {
"mac": "00:50:56:a2:3f:4b"
}
}
{
"id": "cptnod-1",
"name": "OCP 4 – Compute node #2",
"profile": "cptnod",
"selector": {
"mac": "00:50:56:a2:63:85"
}
}
#########################
# 1.) 以上是配置matchbox 自动化部署配置的配置文件,主要需要修改mac地址,需要先确认相应的mac地址的角色,
# 2.) groups中的配置文件“profile”:
“cptnod”,需要与profile中定义的名称一致.
# 启动matchbox
# systemctlenable matchbox && systemctl start matchbox
# matchbox监听8080端口,请确保在防火墙中打开此端口
四. 安装openshift集群
# 创建一对公钥与私钥,用作登录rhcos ,
# ssh-keygen -trsa -b 2048 -N "" -f /root/.ssh/id_ras
# mkdir/root/ocp4
# 创建安装集群配置文件 install-config.yaml
# 请注意修改红色部分
####################
apiVersion: v1
baseDomain:example.com
compute:
-hyperthreading: Enabled
name: worker
replicas: 0
controlPlane:
hyperthreading: Enabled
name: master
replicas: 3
metadata:
name: ocp67
networking:
clusterNetworks:
- cidr: 10.254.0.0/16
hostPrefix: 24
networkType: OpenShiftSDN
serviceNetwork:
- 172.30.0.0/16
platform:
none: {}
pullSecret: '{"auths":{"registry.example.com:5000":
{"auth": "ZHVtbXk6ZHVtbXk=","email":
"noemail@localhost"}}}'
sshKey: '...cat /root/.ssh/id_rsa.pub ... ssh 公钥'
additionalTrustBundle:|
-----BEGIN CERTIFICATE-----
... cat /etc/pki/ca-trust/source/anchors/domain.crt
...私有镜像仓库证书的server.crt
-----END CERTIFICATE-----
imageContentSources:
- mirrors:
-registry.example.com:5000/ocp4/openshift4
source:quay.io/openshift-release-dev/ocp-release
- mirrors:
-registry.example.com:5000/ocp4/openshift4
source:quay.io/openshift-release-dev/ocp-v4.0-art-dev
# 创建集群安装部署配置文件,文件创建之后install-config.yml 文件会被删除,所以建议备份一下.
# openshift-install create ignition-configs
####################
# 复制 *.ign文件到matchbox 的部署目录中
# cp *.ign/var/lib/matchbox/ignition
# 从网卡启动服务器安装openshift 4.2
# 1.) 启动bootstarp-0 机器,安装bootstarp
# openshift-install --dir=/root/ocp4 wait-forbootstrap-complete –log-level debug
# 2.) 等待bootstarp 安装完毕,检查服务是否正常.登录bootstarp,检查服务端口6443,22623.
# 3.) 启动 master-0-3 安装,master,etcd 集群.
# 4.) 将haproxy 中的bootstarp 负载移除,并重启haproxy,
# oc patch configs.imageregistry.operator.openshift.iocluster \
–type merge \
–patch '{"spec":{"storage":{"emptyDir":{}}}}'
# openshift-install --dir=/root/ocp4 wait-forinstall-complete –log-level debug
# 5.) 启动node节点,来部署node~
#6.) 等待node节点部署完毕，需要在部署机，批准node节点申请集群证书的请求
# oc get csr
# 没有批准的证书申请请求会处于pending状态，请确认。
# oc adm certificate approve <csr_name>
注意
由于 CSR 会自动轮转，因此请在将机器添加到集群后一小时内批准您的 CSR。如
果没有在一小时内批准，证书将会轮转，每个节点将会存在多个证书。您必须批准
所有这些证书。批准初始 CSR 后，集群的 kube-controller-manager 会自动批准
后续的节点客户端 CSR。您必须实施一个方法来自动批准 kubelet 提供的证书请
求
# 等待所有的operator 上线~
# oc get no
