openshift 巡检脚本

内容如下:

运行脚本后,会自动添加定时任务;
每天5点执行.
会生成markdown格式报告.

#!/bin/bash 

set -e 
set -x 



function _Auto_Crontab_(){
    #Auto Crontab
path_dir=$(cd "$(dirname "${0}")" && pwd)
file=$(basename $0)
abs_file="${path_dir}/${file}"
log_file="${abs_file}.log"

if [[ $(grep -c "${abs_file}" /etc/crontab) -eq 0  ]] && [[ $UID -eq 0 ]]
then
    echo "00 05 * * * ${USER} ${SHELL} ${abs_file} $1 $2 &>> ${log_file} ">> /etc/crontab
fi
}

_Auto_Crontab_


_Report_Dir_="/scripts/check_ocp/$(date +%F)"

test -d "${_Report_Dir_}" || mkdir -p "${_Report_Dir_}"

_Report_file_="${_Report_Dir_}/$(date +%d_%H_%M).md"

>  "${_Report_file_}"



function _Check_Nodes_Services_(){
    local _NODE_SERVICES_=(
                atomic-openshift-node.service
                dnsmasq
                docker
                )

    for _service_ in ${_NODE_SERVICES_[@]};do 
    echo "#### Check Nodes Service : $_service_ "
    echo ' '
    echo '```bash'
        ansible -i /etc/ansible/hosts nodes -m shell -a "systemctl is-active ${_service_}"  |tee -a "${_Report_file_}"
    echo '```'
    echo ' '
    done 
}

function _Check_Masters_Services_(){
    local _MASTER_SERVICES_=(
            atomic-openshift-master-api.service 
            atomic-openshift-master-controllers.service  
            atomic-openshift-node.service
            dnsmasq
            docker
            )

    for _service_ in ${_MASTER_SERVICES_[@]};do 
        echo "#### Check Master Service : $_service_ "
    echo ' '
    echo '```bash'
        ansible -i /etc/ansible/hosts masters -m shell -a "systemctl is-active ${_service_}"  |tee -a "${_Report_file_}"
    echo '```'
    echo ' '
    done 
}


function echo(){
    /usr/bin/echo  $@ |tee -a "${_Report_file_}"
}

function _Print_(){
    echo "#### The Command: $*"
    echo ' '
    echo '```bash'
    eval $* |tee -a "${_Report_file_}"
    echo '```'
    echo ' '
}

function _Check_All_Node_(){
    echo "#### The All Node Number: $( oc get node |grep -c 'hisense.com')"
    _Print_ 'oc get node'
    echo "#### The NotReady Node Number: $(oc get node |grep -c 'NotReady')"
    echo "## Check All Node Desk Use%"
    echo ' '
    echo '```bash'
    ansible all -m shell -a "df -hT 2>/dev/null |grep -E '^/dev' |grep -E '[6-9][0-9]%'" |tee -a "${_Report_file_}"
    echo '```'
    echo ' '
    echo "## Check All Node Time "
    echo ' '
    echo '```bash'
    ansible all -m shell -a "df -hT 2>/dev/null |grep -E '^/dev' |grep -E '[6-9][0-9]%'" |tee -a "${_Report_file_}"
    echo '```'
    echo ' '
}




echo "# Check Hisense OCP Dev Start $(date +%F_%R)" 


_IAM_="$(oc whoami)"

if [[ "${_IAM_}" != 'system:admin' ]];then
    oc login -u system:admin 
fi 
echo "## Login User Is : $(oc whoami)"

echo "## Check Node "

_Check_All_Node_
_Check_Masters_Services_
_Check_Nodes_Services_



_ALL_NODES_=(
    $(oc get node |grep 'node' |awk '{print $1}')
)

echo "##: The Node Resources Use: CPU Requests  CPU Limits  Memory Requests    Memory Limits %: "
    echo ' '
    echo '```bash'
for node in ${_ALL_NODES_[@]};do
    echo "$(oc describe node ${node} |grep -E -A 4 '^Allocated' |grep -E -o '\([0-9]*\%\)' |grep -E -o '[0-9]*')"
done
    echo '```'
    echo ' '



for node in ${_ALL_NODES_[@]};do
    echo "### Check Node $node "
    _Print_ "oc describe node ${node} |grep -E -A 4 '^Allocatable:' "
    _Print_ "oc describe node ${node} |grep -E -A 10000 '^Non-terminated' "
done

报告

-----------------------------------------------------------------------------------------------

Check Hisense OCP Prod Start 2018-11-01_05:00

Login User Is : system:admin

Check Node

The All Node Number: 17

The Command: oc get node

NAME                        STATUS                     ROLES     AGE       VERSION
prdinfranode1.example.com   Ready                      compute   86d       v1.9.1+a0ce1bc657
prdinfranode2.example.com   Ready                      compute   86d       v1.9.1+a0ce1bc657
prdinfranode3.example.com   Ready                      compute   86d       v1.9.1+a0ce1bc657
prdinfranode4.example.com   Ready                      compute   86d       v1.9.1+a0ce1bc657
prdinfranode5.example.com   Ready                      compute   86d       v1.9.1+a0ce1bc657
prdinfranode6.example.com   Ready                      compute   86d       v1.9.1+a0ce1bc657
prdmaster1.example.com      Ready,SchedulingDisabled   master    86d       v1.9.1+a0ce1bc657
prdmaster2.example.com      Ready,SchedulingDisabled   master    86d       v1.9.1+a0ce1bc657
prdmaster3.example.com      Ready,SchedulingDisabled   master    86d       v1.9.1+a0ce1bc657
prdnode1.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode2.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode3.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode4.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode5.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode6.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode7.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657
prdnode8.example.com        Ready                      compute   86d       v1.9.1+a0ce1bc657

The NotReady Node Number: 0

Check All Node Desk Use%

prdnode7.example.com | FAILED | rc=1 >>
non-zero return code
prdnode6.example.com | FAILED | rc=1 >>
non-zero return code
prdnode8.example.com | FAILED | rc=1 >>
non-zero return code
prdinfranode5.example.com | FAILED | rc=1 >>
non-zero return code
prdinfranode4.example.com | FAILED | rc=1 >>
non-zero return code
prdnode1.example.com | FAILED | rc=1 >>
non-zero return code
prdnode2.example.com | FAILED | rc=1 >>
non-zero return code
prdnode4.example.com | FAILED | rc=1 >>
non-zero return code
prdnode3.example.com | FAILED | rc=1 >>
non-zero return code
prdnode5.example.com | FAILED | rc=1 >>
non-zero return code
prdinfranode1.example.com | SUCCESS | rc=0 >>
/dev/mapper/vg--esdata-lv--esdata xfs       500G  298G  203G  60% /esdata
prdinfranode6.example.com | FAILED | rc=1 >>
non-zero return code
prdinfranode2.example.com | SUCCESS | rc=0 >>
/dev/mapper/vg--esdata-lv--esdata xfs       500G  298G  202G  60% /esdata
prdinfranode3.example.com | SUCCESS | rc=0 >>
/dev/mapper/vg--esdata-lv--esdata xfs       500G  298G  202G  60% /esdata
prdmaster1.example.com | FAILED | rc=1 >>
non-zero return code
prdmaster2.example.com | FAILED | rc=1 >>
non-zero return code
prdmaster3.example.com | FAILED | rc=1 >>
non-zero return code

Check Master Service : atomic-openshift-master-api.service

prdmaster3.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster1.example.com | SUCCESS | rc=0 >>
active

Check Master Service : atomic-openshift-master-controllers.service

prdmaster3.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster1.example.com | SUCCESS | rc=0 >>
active

Check Master Service : atomic-openshift-node.service

prdmaster1.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster3.example.com | SUCCESS | rc=0 >>
active

Check Master Service : dnsmasq

prdmaster1.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster3.example.com | SUCCESS | rc=0 >>
active

Check Master Service : docker

prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster3.example.com | SUCCESS | rc=0 >>
active
prdmaster1.example.com | SUCCESS | rc=0 >>
active

Check Nodes Service : atomic-openshift-node.service

prdmaster1.example.com | SUCCESS | rc=0 >>
active
prdinfranode1.example.com | SUCCESS | rc=0 >>
active
prdinfranode2.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster3.example.com | SUCCESS | rc=0 >>
active
prdnode2.example.com | SUCCESS | rc=0 >>
active
prdinfranode3.example.com | SUCCESS | rc=0 >>
active
prdnode1.example.com | SUCCESS | rc=0 >>
active
prdnode3.example.com | SUCCESS | rc=0 >>
active
prdnode4.example.com | SUCCESS | rc=0 >>
active
prdnode5.example.com | SUCCESS | rc=0 >>
active
prdnode6.example.com | SUCCESS | rc=0 >>
active
prdinfranode5.example.com | SUCCESS | rc=0 >>
active
prdinfranode6.example.com | SUCCESS | rc=0 >>
active
prdinfranode4.example.com | SUCCESS | rc=0 >>
active
prdnode7.example.com | SUCCESS | rc=0 >>
active
prdnode8.example.com | SUCCESS | rc=0 >>
active

Check Nodes Service : dnsmasq

prdinfranode1.example.com | SUCCESS | rc=0 >>
active
prdinfranode2.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster3.example.com | SUCCESS | rc=0 >>
active
prdmaster1.example.com | SUCCESS | rc=0 >>
active
prdinfranode3.example.com | SUCCESS | rc=0 >>
active
prdnode1.example.com | SUCCESS | rc=0 >>
active
prdnode4.example.com | SUCCESS | rc=0 >>
active
prdnode2.example.com | SUCCESS | rc=0 >>
active
prdnode3.example.com | SUCCESS | rc=0 >>
active
prdnode5.example.com | SUCCESS | rc=0 >>
active
prdnode6.example.com | SUCCESS | rc=0 >>
active
prdinfranode6.example.com | SUCCESS | rc=0 >>
active
prdinfranode4.example.com | SUCCESS | rc=0 >>
active
prdinfranode5.example.com | SUCCESS | rc=0 >>
active
prdnode7.example.com | SUCCESS | rc=0 >>
active
prdnode8.example.com | SUCCESS | rc=0 >>
active

Check Nodes Service : docker

prdinfranode1.example.com | SUCCESS | rc=0 >>
active
prdinfranode2.example.com | SUCCESS | rc=0 >>
active
prdmaster2.example.com | SUCCESS | rc=0 >>
active
prdmaster3.example.com | SUCCESS | rc=0 >>
active
prdmaster1.example.com | SUCCESS | rc=0 >>
active
prdnode2.example.com | SUCCESS | rc=0 >>
active
prdnode1.example.com | SUCCESS | rc=0 >>
active
prdinfranode3.example.com | SUCCESS | rc=0 >>
active
prdnode3.example.com | SUCCESS | rc=0 >>
active
prdnode4.example.com | SUCCESS | rc=0 >>
active
prdnode5.example.com | SUCCESS | rc=0 >>
active
prdnode6.example.com | SUCCESS | rc=0 >>
active
prdinfranode6.example.com | SUCCESS | rc=0 >>
active
prdinfranode4.example.com | SUCCESS | rc=0 >>
active
prdinfranode5.example.com | SUCCESS | rc=0 >>
active
prdnode7.example.com | SUCCESS | rc=0 >>
active
prdnode8.example.com | SUCCESS | rc=0 >>
active

: The Node Resources Use: CPU Requests CPU Limits Memory Requests Memory Limits %:

21 0 38 48
17 0 36 40
20 0 39 44
2 0 9 6
2 0 9 6
2 0 2 1
5 0 3 3
5 0 3 3
5 0 3 3
25 87 62 119
25 87 54 111
22 87 60 119
6 25 18 34
25 87 54 111
1 0 1 1
1 0 1 1
1 0 1 1

Check Node prdinfranode1.example.com

The Command: oc describe node prdinfranode1.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdinfranode1.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (8 in total)
  Namespace                  Name                                       CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                       ------------  ----------  ---------------  -------------
  default                    docker-registry-2-hx8vl                    100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  default                    router-1-lfktr                             100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  logging                    logging-curator-13-9xdgq                   100m (1%)     0 (0%)      256Mi (0%)       256Mi (0%)
  logging                    logging-es-data-master-iv056e3b-4-2nbff    1100m (13%)   0 (0%)      8256Mi (25%)     8256Mi (25%)
  logging                    logging-fluentd-qmlsk                      100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
  logging                    logging-kibana-2-dcw9r                     200m (2%)     0 (0%)      992Mi (3%)       992Mi (3%)
  openshift-infra            hawkular-cassandra-1-mlmgr                 0 (0%)        0 (0%)      1G (2%)          2G (5%)
  openshift-infra            heapster-s7n2c                             0 (0%)        0 (0%)      937500k (2%)     3750M (11%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests    Memory Limits
  ------------  ----------  ---------------    -------------
  1700m (21%)   0 (0%)      12976908128 (38%)  16252537216 (48%)
Events:         <none>

Check Node prdinfranode2.example.com

The Command: oc describe node prdinfranode2.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdinfranode2.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (6 in total)
  Namespace                  Name                                       CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                       ------------  ----------  ---------------  -------------
  default                    docker-registry-2-4cmmx                    100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  default                    router-1-nw5ck                             100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  logging                    logging-es-data-master-ryom5kjr-4-86zjx    1100m (13%)   0 (0%)      8256Mi (25%)     8256Mi (25%)
  logging                    logging-fluentd-l9zgr                      100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
  openshift-infra            hawkular-cassandra-2-4q7bw                 0 (0%)        0 (0%)      1G (2%)          2G (5%)
  openshift-infra            hawkular-metrics-2zkrq                     0 (0%)        0 (0%)      1500M (4%)       2500M (7%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests    Memory Limits
  ------------  ----------  ---------------    -------------
  1400m (17%)   0 (0%)      12230785280 (36%)  13693914368 (40%)
Events:         <none>

Check Node prdinfranode3.example.com

The Command: oc describe node prdinfranode3.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdinfranode3.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (7 in total)
  Namespace                  Name                                       CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                       ------------  ----------  ---------------  -------------
  default                    docker-registry-2-jm7sj                    100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  default                    router-1-qz4lt                             100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  logging                    logging-es-data-master-tc5e4et1-4-6lt7q    1100m (13%)   0 (0%)      8256Mi (25%)     8256Mi (25%)
  logging                    logging-fluentd-xx847                      100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
  logging                    logging-kibana-2-2547z                     200m (2%)     0 (0%)      992Mi (3%)       992Mi (3%)
  openshift-infra            hawkular-cassandra-3-8qlbz                 0 (0%)        0 (0%)      1G (2%)          2G (5%)
  openshift-infra            hawkular-metrics-hzr98                     0 (0%)        0 (0%)      1500M (4%)       2500M (7%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests    Memory Limits
  ------------  ----------  ---------------    -------------
  1600m (20%)   0 (0%)      13270972672 (39%)  14734101760 (44%)
Events:         <none>

Check Node prdinfranode4.example.com

The Command: oc describe node prdinfranode4.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  7889928Ki
 pods:    80
System Info:

The Command: oc describe node prdinfranode4.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (2 in total)
  Namespace                  Name                     CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                     ------------  ----------  ---------------  -------------
  default                    routerdmz-3-55fs6        100m (1%)     0 (0%)      256Mi (3%)       0 (0%)
  logging                    logging-fluentd-m9wgb    100m (1%)     0 (0%)      512Mi (6%)       512Mi (6%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  200m (2%)     0 (0%)      768Mi (9%)       512Mi (6%)
Events:         <none>

Check Node prdinfranode5.example.com

The Command: oc describe node prdinfranode5.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  7889924Ki
 pods:    80
System Info:

The Command: oc describe node prdinfranode5.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (2 in total)
  Namespace                  Name                     CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                     ------------  ----------  ---------------  -------------
  default                    routerdmz-3-brjx8        100m (1%)     0 (0%)      256Mi (3%)       0 (0%)
  logging                    logging-fluentd-2kdck    100m (1%)     0 (0%)      512Mi (6%)       512Mi (6%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  200m (2%)     0 (0%)      768Mi (9%)       512Mi (6%)
Events:         <none>

Check Node prdinfranode6.example.com

The Command: oc describe node prdinfranode6.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdinfranode6.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (2 in total)
  Namespace                  Name                     CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                     ------------  ----------  ---------------  -------------
  default                    routerdmz-3-vpb8x        100m (1%)     0 (0%)      256Mi (0%)       0 (0%)
  logging                    logging-fluentd-c78zq    100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  200m (2%)     0 (0%)      768Mi (2%)       512Mi (1%)
Events:         <none>

Check Node prdmaster1.example.com

The Command: oc describe node prdmaster1.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     4
 memory:  16147424Ki
 pods:    40
System Info:

The Command: oc describe node prdmaster1.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (3 in total)
  Namespace                  Name                           CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                           ------------  ----------  ---------------  -------------
  default                    registry-console-1-zsktg       0 (0%)        0 (0%)      0 (0%)           0 (0%)
  logging                    logging-fluentd-tq4cx          100m (2%)     0 (0%)      512Mi (3%)       512Mi (3%)
  openshift-web-console      webconsole-759b44bbdd-5jh7g    100m (2%)     0 (0%)      100Mi (0%)       0 (0%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  200m (5%)     0 (0%)      612Mi (3%)       512Mi (3%)
Events:         <none>

Check Node prdmaster2.example.com

The Command: oc describe node prdmaster2.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     4
 memory:  16147440Ki
 pods:    40
System Info:

The Command: oc describe node prdmaster2.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (2 in total)
  Namespace                  Name                           CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                           ------------  ----------  ---------------  -------------
  logging                    logging-fluentd-z94ht          100m (2%)     0 (0%)      512Mi (3%)       512Mi (3%)
  openshift-web-console      webconsole-759b44bbdd-5xlkz    100m (2%)     0 (0%)      100Mi (0%)       0 (0%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  200m (5%)     0 (0%)      612Mi (3%)       512Mi (3%)
Events:         <none>

Check Node prdmaster3.example.com

The Command: oc describe node prdmaster3.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     4
 memory:  16147440Ki
 pods:    40
System Info:

The Command: oc describe node prdmaster3.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (2 in total)
  Namespace                  Name                           CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                           ------------  ----------  ---------------  -------------
  logging                    logging-fluentd-7hqm7          100m (2%)     0 (0%)      512Mi (3%)       512Mi (3%)
  openshift-web-console      webconsole-759b44bbdd-49jxl    100m (2%)     0 (0%)      100Mi (0%)       0 (0%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  200m (5%)     0 (0%)      612Mi (3%)       512Mi (3%)
Events:         <none>

Check Node prdnode1.example.com

The Command: oc describe node prdnode1.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     16
 memory:  49438616Ki
 pods:    160
System Info:

The Command: oc describe node prdnode1.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (9 in total)
  Namespace                  Name                                                      CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                                      ------------  ----------  ---------------  -------------
  hisense-ecp                hisense-ecp-front-19-hjp67                                1 (6%)        2 (12%)     1Gi (2%)         4Gi (8%)
  hisense-ecp                hisense-inventory-gateway-erp-starter-18-d5jfs            500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-gateway-retail-starter-11-swmxh         500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-inventory-inventory-starter-19-nz5s7    500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-oms-rest-starter-15-vmb2c               500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-order-order-starter-11-rn94x            500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-user-usercenter-starter-13-5gphj        500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                jenkins-1-847s4                                           0 (0%)        0 (0%)      4Gi (8%)         4Gi (8%)
  logging                    logging-fluentd-4kbgd                                     100m (0%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  4100m (25%)   14 (87%)    30208Mi (62%)    57856Mi (119%)
Events:         <none>

Check Node prdnode2.example.com

The Command: oc describe node prdnode2.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     16
 memory:  49438616Ki
 pods:    160
System Info:

The Command: oc describe node prdnode2.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (8 in total)
  Namespace                  Name                                                  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                                  ------------  ----------  ---------------  -------------
  hisense-ecp                hisense-ecp-front-19-jb7ph                            1 (6%)        2 (12%)     1Gi (2%)         4Gi (8%)
  hisense-ecp                hisense-inventory-item-item-starter-17-6h2j9          500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-item-item-starter-17-khgt6          500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-oms-admin-starter-15-5q5v2          500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-oms-rest-starter-15-bqtnr           500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-order-order-starter-11-4ppsg        500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-user-usercenter-starter-13-z5gnt    500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  logging                    logging-fluentd-dx64t                                 100m (0%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  4100m (25%)   14 (87%)    26112Mi (54%)    53760Mi (111%)
Events:         <none>

Check Node prdnode3.example.com

The Command: oc describe node prdnode3.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     16
 memory:  49438632Ki
 pods:    160
System Info:

The Command: oc describe node prdnode3.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (8 in total)
  Namespace                  Name                                                      CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                                      ------------  ----------  ---------------  -------------
  hisense-ecp                hisense-inventory-gateway-erp-starter-18-glgdl            500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-gateway-retail-starter-11-njdtf         500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-inventory-inventory-starter-19-smx64    500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-oms-rest-starter-15-87fks               500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-order-order-starter-11-84tdw            500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-scheduler-starter-10-9wfx5              500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-user-usercenter-starter-13-xm4hm        500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  logging                    logging-fluentd-mkp92                                     100m (0%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  3600m (22%)   14 (87%)    29184Mi (60%)    57856Mi (119%)
Events:         <none>

Check Node prdnode4.example.com

The Command: oc describe node prdnode4.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     16
 memory:  49438616Ki
 pods:    160
System Info:

The Command: oc describe node prdnode4.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (3 in total)
  Namespace                  Name                                              CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                              ------------  ----------  ---------------  -------------
  hisense-ecp                hisense-inventory-gateway-erp-starter-18-n6rmm    500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-scheduler-starter-10-hwgjc      500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  logging                    logging-fluentd-vgz9l                             100m (0%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  1100m (6%)    4 (25%)     8704Mi (18%)     16896Mi (34%)
Events:         <none>

Check Node prdnode5.example.com

The Command: oc describe node prdnode5.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     16
 memory:  49438616Ki
 pods:    160
System Info:

The Command: oc describe node prdnode5.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (8 in total)
  Namespace                  Name                                                      CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                                                      ------------  ----------  ---------------  -------------
  hisense-ecp                hisense-ecp-front-19-n8qcf                                1 (6%)        2 (12%)     1Gi (2%)         4Gi (8%)
  hisense-ecp                hisense-inventory-gateway-retail-starter-11-qszq7         500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-inventory-inventory-starter-19-jjhm9    500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-item-item-starter-17-kn5wk              500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-oms-admin-starter-15-5fknd              500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-oms-admin-starter-15-5g2ts              500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  hisense-ecp                hisense-inventory-scheduler-starter-10-p2w6h              500m (3%)     2 (12%)     4Gi (8%)         8Gi (16%)
  logging                    logging-fluentd-lfpp4                                     100m (0%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  4100m (25%)   14 (87%)    26112Mi (54%)    53760Mi (111%)
Events:         <none>

Check Node prdnode6.example.com

The Command: oc describe node prdnode6.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdnode6.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (1 in total)
  Namespace                  Name                     CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                     ------------  ----------  ---------------  -------------
  logging                    logging-fluentd-9g57w    100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Events:         <none>

Check Node prdnode7.example.com

The Command: oc describe node prdnode7.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdnode7.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (1 in total)
  Namespace                  Name                     CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                     ------------  ----------  ---------------  -------------
  logging                    logging-fluentd-7d8fx    100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Events:         <none>

Check Node prdnode8.example.com

The Command: oc describe node prdnode8.example.com |grep -E -A 4 '^Allocatable:'

Allocatable:
 cpu:     8
 memory:  32661400Ki
 pods:    80
System Info:

The Command: oc describe node prdnode8.example.com |grep -E -A 10000 '^Non-terminated'

Non-terminated Pods:         (1 in total)
  Namespace                  Name                     CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ---------                  ----                     ------------  ----------  ---------------  -------------
  logging                    logging-fluentd-pvgwk    100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  CPU Requests  CPU Limits  Memory Requests  Memory Limits
  ------------  ----------  ---------------  -------------
  100m (1%)     0 (0%)      512Mi (1%)       512Mi (1%)
Events:         <none>

最后编辑于：2018.11.01 17:13:33

人面猴
序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
沈念sama阅读 158,560评论 4赞 361
死咒
序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
沈念sama阅读 67,104评论 1赞 291
救了他两次的神仙让他今天三更去死
文/潘晓璐我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
开封第一讲书人阅读 108,297评论 0赞 243
道士缉凶录：失踪的卖姜人
文/不坏的土叔我叫张陵，是天一观的道长。经常有香客问我，道长，这世上最难降的妖魔是什么？我笑而不...
开封第一讲书人阅读 43,869评论 0赞 204
港岛之恋（遗憾婚礼）
正文为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
茶点故事阅读 52,275评论 3赞 287
恶毒庶女顶嫁案：这布局不是一般人想出来的
文/花漫我一把揭开白布。她就那样静静地躺着，像睡着了一般。火红的嫁衣衬着肌肤如雪。梳的纹丝不乱的头发上，一...
开封第一讲书人阅读 40,563评论 1赞 216
城市分裂传说
那天，我揣着相机与录音，去河边找鬼。笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。我是一名探鬼主播，决...
沈念sama阅读 31,833评论 2赞 312
双鸳鸯连环套：你想象不到人心有多黑
文/苍兰香墨我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
开封第一讲书人阅读 30,543评论 0赞 197
万荣杀人案实录
序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
沈念sama阅读 34,245评论 1赞 241
护林员之死
正文独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋以下内容为张勋视角年9月15日...
茶点故事阅读 30,512评论 2赞 244
白月光启示录
正文我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
茶点故事阅读 32,011评论 1赞 258
活死人
序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
沈念sama阅读 28,359评论 2赞 253
日本核电站爆炸内幕
正文年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
茶点故事阅读 33,006评论 3赞 235
男人毒药：我在死后第九天来索命
文/蒙蒙一、第九天我趴在偏房一处隐蔽的房顶上张望。院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
开封第一讲书人阅读 26,062评论 0赞 8
一桩弑父案，背后竟有这般阴谋
文/苍兰香墨我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。一阵脚步声响...
开封第一讲书人阅读 26,825评论 0赞 194
情欲美人皮
我被黑心中介骗来泰国打工，没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。一个月前我还...
沈念sama阅读 35,590评论 2赞 273
代替公主和亲
正文我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
茶点故事阅读 35,501评论 2赞 268

openshift 巡检脚本

内容如下:

报告

-----------------------------------------------------------------------------------------------

Check Hisense OCP Prod Start 2018-11-01_05:00

Login User Is : system:admin

Check Node

The All Node Number: 17

The Command: oc get node

The NotReady Node Number: 0

Check All Node Desk Use%

Check Master Service : atomic-openshift-master-api.service

Check Master Service : atomic-openshift-master-controllers.service

Check Master Service : atomic-openshift-node.service

Check Master Service : dnsmasq

Check Master Service : docker

Check Nodes Service : atomic-openshift-node.service

Check Nodes Service : dnsmasq

Check Nodes Service : docker

: The Node Resources Use: CPU Requests CPU Limits Memory Requests Memory Limits %:

Check Node prdinfranode1.example.com

The Command: oc describe node prdinfranode1.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdinfranode1.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdinfranode2.example.com

The Command: oc describe node prdinfranode2.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdinfranode2.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdinfranode3.example.com

The Command: oc describe node prdinfranode3.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdinfranode3.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdinfranode4.example.com

The Command: oc describe node prdinfranode4.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdinfranode4.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdinfranode5.example.com

The Command: oc describe node prdinfranode5.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdinfranode5.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdinfranode6.example.com

The Command: oc describe node prdinfranode6.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdinfranode6.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdmaster1.example.com

The Command: oc describe node prdmaster1.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdmaster1.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdmaster2.example.com

The Command: oc describe node prdmaster2.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdmaster2.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdmaster3.example.com

The Command: oc describe node prdmaster3.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdmaster3.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode1.example.com

The Command: oc describe node prdnode1.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode1.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode2.example.com

The Command: oc describe node prdnode2.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode2.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode3.example.com

The Command: oc describe node prdnode3.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode3.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode4.example.com

The Command: oc describe node prdnode4.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode4.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode5.example.com

The Command: oc describe node prdnode5.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode5.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode6.example.com

The Command: oc describe node prdnode6.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode6.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode7.example.com

The Command: oc describe node prdnode7.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode7.example.com |grep -E -A 10000 '^Non-terminated'

Check Node prdnode8.example.com

The Command: oc describe node prdnode8.example.com |grep -E -A 4 '^Allocatable:'

The Command: oc describe node prdnode8.example.com |grep -E -A 10000 '^Non-terminated'

推荐阅读更多精彩内容