本文参考openshift 3.11离线安装，遇到了一些环境相关的问题，所以梳理了以下，如果遇到类似问题，可以参考。
环境准备
- 硬件环境
公司内部虚拟机3台 host-10-1-236-72, host-10-1-236-73, host-10-1-236-77
操作系统 ：CentOS 7.4
[root@host-10-1-236-72 ~]# cat /etc/redhat-release
CentOS Linux release 7.4.1708 (Core)
- 手动配置/etc/hosts
[root@host-10-1-236-72 ~]# cat /etc/hosts
10.1.236.72 host-10-1-236-72
10.1.236.73 host-10-1-236-73
10.1.236.77 host-10-1-236-77
- 节点安排
- master节点：host-10-1-236-72
- infra节点：host-10-1-236-73
- compute节点：host-10-1-236-77
- 在所有节点上设置selinux
[root@host-10-1-236-72 ~]# cat /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=enforcing
#SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected.
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
- 离线安装必须要准备安装中依赖的yum源
源标识 源名称 状态
atomic atomic 12
centos-openshift-origin311 CentOS OpenShift Origin 60
ceph-common ceph-common 14
container-selinux container-selinux 18
docker docker 17
epel epel 1
extras extras 434
python-docker python-docker 4
- 节点间设置ssh免密访问
ssh-keygen -f ~/.ssh/id_rsa -N ''
for host in host-10-1-236-73 host-10-1-236-77
do
ssh-copy-id -i ~/.ssh/id_rsa.pub $host;
done
- 安装iptables，并配置iptables
yum install iptables-services -y
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 81 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;systemctl enable iptables
Tips
由于Opshift 使用管理节点的8443端口，因此除了上述端口加入iptables外，还需要将8443端口加入到iptables，否则在安装后加入nodes时，无法加入。
-A INPUT -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT
-A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT
- 找一台机器作为ansible执行的机器，安装openshift ansible
yum install openshift-ansible
- 配置ansile hosts
[root@host-10-1-236-77 ~]# cat /etc/ansible/hosts
# Create an OSEv3 group that contains the masters, nodes, and etcd groups
[OSEv3:children]
masters
nodes
etcd
# Set variables common for all OSEv3 hosts
[OSEv3:vars]
# SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root
#openshift_deployment_type=openshift-enterprise
openshift_deployment_type=origin
openshift_release="3.11"
openshift_image_tag=v3.11
openshift_pkg_version=-3.11.0
openshift_use_openshift_sdn=true
# If ansible_ssh_user is not root, ansible_become must be set to true
#ansible_become=true
#containerized=false
# default selectors for router and registry services
# openshift_router_selector='node-role.kubernetes.io/infra=true'
# openshift_registry_selector='node-role.kubernetes.io/infra=true'
# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
#openshift_master_default_subdomain=ai.com
openshift_disable_check=memory_availability,disk_availability,docker_image_availability
os_sdn_network_plugin_name='redhat/openshift-ovs-networkpolicy'
openshift_master_cluster_method=native
openshift_master_cluster_hostname=host-10-1-236-72
openshift_master_cluster_public_hostname=host-10-1-236-72
# false
ansible_service_broker_install=false
openshift_enable_service_catalog=false
template_service_broker_install=false
openshift_logging_install_logging=false
enable_excluders=false
# registry passwd
#oreg_url=10.1.236.77:5000/openshift3/ose-${component}:${version}
oreg_url=10.1.236.77:5000/openshift/origin-${component}:${version}
openshift_examples_modify_imagestreams=true
# docker config
openshift_docker_additional_registries=10.1.236.77:5000
openshift_docker_insecure_registries=10.1.236.77:5000
#openshift_docker_blocked_registries
openshift_docker_options="--log-driver json-file --log-opt max-size=1M --log-opt max-file=3"
# openshift_cluster_monitoring_operator_install=false
# openshift_metrics_install_metrics=true
# openshift_enable_unsupported_configurations=True
#openshift_logging_es_nodeselector='node-role.kubernetes.io/infra: "true"'
#openshift_logging_kibana_nodeselector='node-role.kubernetes.io/infra: "true"'
# host group for masters
[masters]
host-10-1-236-72
# host group for etcd
[etcd]
host-10-1-236-72
# host group for nodes, includes region info
[nodes]
host-10-1-236-72 openshift_node_group_name='node-config-master'
host-10-1-236-73 openshift_node_group_name='node-config-compute'
host-10-1-236-77 openshift_node_group_name='node-config-compute'
host-10-1-236-73 openshift_node_group_name='node-config-infra'
- 安装docker
yum install docker
- 找一台机器作为docker镜像仓库，安装docker-distribution
本环境选 host-10-1-236-77节点安装docker镜像仓库
yum install -y docker-distribution;systemctl start docker-distribution;systemctl enable docker-distribution
- 配置docker /etc/containers/registries.conf，修改镜像仓库地址。分发到各节点并重启docker
[root@host-10-1-236-77 ~]# cat /etc/containers/registries.conf
# This is a system-wide configuration file used to
# keep track of registries for various container backends.
# It adheres to TOML format and does not support recursive
# lists of registries.
# The default location for this configuration file is /etc/containers/registries.conf.
# The only valid categories are: 'registries.search', 'registries.insecure',
# and 'registries.block'.
[registries.search]
registries = ['10.1.236.77:5000']
# If you need to access insecure registries, add the registry's fully-qualified name.
# An insecure registry is one that does not have a valid SSL certificate or only does HTTP.
[registries.insecure]
registries = ['10.1.236.77:5000']
# If you need to block pull access from a registry, uncomment the section below
# and add the registries fully-qualified name.
#
# Docker only
[registries.block]
registries = [registry.access.redhat.com]
ansible all -m copy -a "src=/etc/containers/registries.conf dest=/etc/containers/registries.conf "
ansible all -a 'systemctl start docker';ansible all -a 'systemctl enable docker'
- 下载如下docker镜像，并上传到私有镜像仓库中.(离线安装使用)
[root@host-10-1-236-77 data]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
docker.io/openshift/origin-node v3.11 32492bc114ab 4 weeks ago 1.16 GB
docker.io/openshift/origin-control-plane v3.11 bf9dd3be958f 4 weeks ago 818 MB
docker.io/openshift/origin-deployer v3.11.0 c3351a73e321 4 weeks ago 376 MB
docker.io/openshift/origin-haproxy-router v3.11 88f17a56a362 4 weeks ago 401 MB
docker.io/openshift/origin-pod v3.11.0 947fb5afa0ce 4 weeks ago 254 MB
docker.io/openshift/origin-web-console v3.11 be30b6cce5fa 6 weeks ago 339 MB
docker.io/openshift/origin-docker-registry v3.11 a0d5ad164395 6 weeks ago 305 MB
docker.io/openshift/origin-metrics-server v3.11 8c99f32f40d3 6 weeks ago 311 MB
docker.io/openshift/origin-console v3.11 c2b807836120 6 weeks ago 260 MB
docker.io/openshift/origin-metrics-heapster v3.11 69421c019449 6 weeks ago 710 MB
docker.io/openshift/origin-metrics-hawkular-metrics v3.11 59e2258250c4 6 weeks ago 860 MB
docker.io/openshift/origin-metrics-schema-installer v3.11 342f50fded7d 6 weeks ago 551 MB
docker.io/openshift/origin-metrics-cassandra v3.11 8176cfabc16b 6 weeks ago 590 MB
docker.io/cockpit/kubernetes latest 4ec8c5b8a486 6 weeks ago 336 MB
quay.io/coreos/cluster-monitoring-operator v0.1.1 4488a207a5bc 2 months ago 510 MB
quay.io/coreos/prometheus-config-reloader v0.23.2 2ed5973a47af 3 months ago 12.2 MB
quay.io/coreos/prometheus-operator v0.23.2 835a7e260b35 3 months ago 47 MB
docker.io/openshift/prometheus-alertmanager v0.15.2 68bbd0006378 3 months ago 233 MB
docker.io/openshift/prometheus-node-exporter v0.16.0 f9f775bf6d0e 4 months ago 216 MB
docker.io/openshift/prometheus v2.3.2 e362c322f000 4 months ago 316 MB
docker.io/grafana/grafana 5.2.1 1bfead9ff707 5 months ago 245 MB
quay.io/coreos/kube-rbac-proxy v0.3.1 992ac1a5e7c7 5 months ago 40.2 MB
quay.io/coreos/etcd v3.2.22 ff5dd2137a4f 5 months ago 37.3 MB
quay.io/coreos/kube-state-metrics v1.3.1 a9c8f313b7aa 7 months ago 22.2 MB
docker.io/openshift/oauth-proxy v1.1.0 90c45954eb03 9 months ago 235 MB
quay.io/coreos/configmap-reload v0.0.1 3129a2ca29d7 23 months ago 4.79 MB
给镜像打私有仓库的tag并push到私有仓库中。
docker images|grep docker.io/openshift/origin-node|awk '{print $1"/"$2}'|awk -F "/" '{print "docker tag "$1"/"$2"/"$3":"$4" 10.1.236.77:5000/"$2"/"$3":"$4}'|sh
docker images|grep 10.1.236.77:5000|awk '{print "docker push " $1":"$2}'|sh
- 使用ansible book预检查openshift
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml
Tips
- 在预检查过程中，如果报错，可以通过jourctl查看错误原因，如果是因为镜像不全，需要按照#13步操作方式，离线下载缺少的镜像并上传到私有仓库
- 预检查其中有一个task会自动将CentOS-OpenShift-Origin.repo更新为公网的repo，会将私有repo替换掉，导致预检查失败。在检查中，可以手动替换，这样retry就会过去。
- 使用ansible book安装openshift
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml -vvv
- 如果安装过程中出现问题，也可以通过ansible清除下，再重新安装，命令如下。
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml
Tips
删除后，必须要手动将ip link删除，否则再次安装时，安装失败，报错"OKD install fails: network plugin is not ready: cni config uninitialize on default SDN"
查看SDN pods日志SDN pod拉起时就会报错. "SDN node startup failed: node SDN setup failed: error on port vxlan0: "could not add network device vxlan0 to ofproto (File exists)"
遇到这种问题时，在节点上使用ip link来查询并删除，操作命令如下：
ip link delete vxlan_sys_4789
- 安装结束后创建用户
htpasswd -cb /etc/origin/master/htpasswd admin mypwd
oc adm policy add-cluster-role-to-user cluster-admin admin
- 部署后检查
- 后台操作检查
检查节点是否ready
[root@host-10-1-236-72 ~]# oc get nodes
NAME STATUS ROLES AGE VERSION
host-10-1-236-72 Ready master 6d v1.11.0+d4cacc0
host-10-1-236-73 Ready infra 6d v1.11.0+d4cacc0
host-10-1-236-77 Ready compute 6d v1.11.0+d4cacc0
检查projects
[root@host-10-1-236-72 ~]# oc projects
You have access to the following projects and can switch between them with 'oc project <projectname>':
* default
kube-public
kube-system
management-infra
openshift
openshift-console
openshift-infra
openshift-logging
openshift-monitoring
openshift-node
openshift-sdn
openshift-web-console
- 前台操作检查
浏览器打开链接https://host-10-1-236-72:8443 出现登录界面。login
使用admin用户登录系统
dashboard
Tips
oc logs无法查看pods日志，报错："ipaddress:10250: connect: no route to host"。原因在于需要放开子节点10250端口的iptables。操作如下，检查/etc/sysconfig/iptables，如果10250端口没有被加进去，添加10250的策略
-A INPUT -p tcp -m state --state NEW -m tcp --dport 10250 -j ACCEPT
-A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 10250 -j ACCEPT
