操作系统信息
虚拟机，Centos7 三个节点 1Master 2node
Kubernetes版本信息
[root@k8sMaster2 test]# kubectl version
Client Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.3", GitCommit:"06ad960bfd03b39c8310aaf92d1e7c12ce618213", GitTreeState:"clean", BuildDate:"2020-02-11T18:14:22Z", GoVersion:"go1.13.6", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.3", GitCommit:"06ad960bfd03b39c8310aaf92d1e7c12ce618213", GitTreeState:"clean", BuildDate:"2020-02-11T18:07:13Z", GoVersion:"go1.13.6", Compiler:"gc", Platform:"linux/amd64"}
最小化安装时ks-account和ks-apigateway一直失败
查看pod运行信息
[root@k8sMaster2 test]# kubectl get pods --all-namespaces -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default dnsutils 1/1 Running 0 38m 10.244.2.31 k8snode1 <none> <none>
default example1-helm-chart-fdb7c764-442sx 1/1 Running 29 22h 10.244.0.34 k8smaster2 <none> <none>
default tomcat6-5f7ccf4cb9-q56vc 1/1 Running 1 82d 10.244.1.16 k8snode2 <none> <none>
kube-system coredns-7f9c544f75-f554c 1/1 Running 0 16m 10.244.0.98 k8smaster2 <none> <none>
kube-system coredns-7f9c544f75-tzhm6 1/1 Running 0 16m 10.244.1.27 k8snode2 <none> <none>
kube-system etcd-k8smaster2 1/1 Running 1 83d 192.168.149.4 k8smaster2 <none> <none>
kube-system kube-apiserver-k8smaster2 1/1 Running 2 83d 192.168.149.4 k8smaster2 <none> <none>
kube-system kube-controller-manager-k8smaster2 1/1 Running 1 83d 192.168.149.4 k8smaster2 <none> <none>
kube-system kube-flannel-ds-amd64-9f9g9 1/1 Running 1 82d 192.168.149.5 k8snode1 <none> <none>
kube-system kube-flannel-ds-amd64-b456t 1/1 Running 1 82d 192.168.149.6 k8snode2 <none> <none>
kube-system kube-flannel-ds-amd64-lzvk9 1/1 Running 1 82d 192.168.149.4 k8smaster2 <none> <none>
kube-system kube-proxy-pmqc6 1/1 Running 1 82d 192.168.149.6 k8snode2 <none> <none>
kube-system kube-proxy-rgf6h 1/1 Running 1 82d 192.168.149.5 k8snode1 <none> <none>
kube-system kube-proxy-zcsts 1/1 Running 1 83d 192.168.149.4 k8smaster2 <none> <none>
kube-system kube-scheduler-k8smaster2 1/1 Running 1 83d 192.168.149.4 k8smaster2 <none> <none>
kube-system tiller-deploy-5fdc6844fb-7qjv9 1/1 Running 1 22h 10.244.2.22 k8snode1 <none> <none>
kubesphere-controls-system default-http-backend-5d464dd566-r2mdm 1/1 Running 2 18h 10.244.1.20 k8snode2 <none> <none>
kubesphere-controls-system kubectl-admin-6c664db975-w47db 1/1 Running 0 12h 10.244.2.28 k8snode1 <none> <none>
kubesphere-monitoring-system kube-state-metrics-566cdbcb48-s9mq2 4/4 Running 0 18h 10.244.0.28 k8smaster2 <none> <none>
kubesphere-monitoring-system node-exporter-4rdqc 2/2 Running 0 18h 192.168.149.6 k8snode2 <none> <none>
kubesphere-monitoring-system node-exporter-v5bhb 2/2 Running 0 18h 192.168.149.5 k8snode1 <none> <none>
kubesphere-monitoring-system node-exporter-zf64s 2/2 Running 1 18h 192.168.149.4 k8smaster2 <none> <none>
kubesphere-monitoring-system prometheus-k8s-0 3/3 Running 1 18h 10.244.1.23 k8snode2 <none> <none>
kubesphere-monitoring-system prometheus-k8s-system-0 3/3 Running 1 18h 10.244.2.26 k8snode1 <none> <none>
kubesphere-monitoring-system prometheus-operator-6b97679cfd-hqf6h 1/1 Running 0 18h 10.244.1.18 k8snode2 <none> <none>
kubesphere-system ks-account-75cb9794bd-wpgvf 0/1 CrashLoopBackOff 6 15m 10.244.0.101 k8smaster2 <none> <none>
kubesphere-system ks-apigateway-79b78f9648-pcgfr 0/1 CrashLoopBackOff 7 15m 10.244.0.100 k8smaster2 <none> <none>
kubesphere-system ks-apiserver-746cc97b9b-fzbgz 1/1 Running 0 15m 10.244.0.99 k8smaster2 <none> <none>
kubesphere-system ks-console-755c8654d9-4p9f9 1/1 Running 0 15m 10.244.0.102 k8smaster2 <none> <none>
kubesphere-system ks-controller-manager-6fd456dc49-rf5m7 1/1 Running 0 15m 10.244.0.103 k8smaster2 <none> <none>
kubesphere-system ks-installer-7d9fb945c7-stsxk 1/1 Running 0 15m 10.244.2.32 k8snode1 <none> <none>
kubesphere-system openldap-0 1/1 Running 0 12h 10.244.0.79 k8smaster2 <none> <none>
kubesphere-system redis-6fd6c6d6f9-54c4n 1/1 Running 0 147m 10.244.0.92 k8smaster2 <none> <none>
openebs openebs-admission-server-5cf6864fbf-mwzt2 1/1 Running 1 20h 10.244.2.20 k8snode1 <none> <none>
openebs openebs-apiserver-bc55cd99b-4x8t6 1/1 Running 1 20h 10.244.0.36 k8smaster2 <none> <none>
openebs openebs-localpv-provisioner-85ff89dd44-przj8 1/1 Running 2 20h 10.244.0.38 k8smaster2 <none> <none>
openebs openebs-ndm-9qgk2 1/1 Running 1 20h 192.168.149.4 k8smaster2 <none> <none>
openebs openebs-ndm-operator-87df44d9-tbpl4 1/1 Running 1 20h 10.244.1.19 k8snode2 <none> <none>
openebs openebs-ndm-r866g 1/1 Running 1 20h 192.168.149.6 k8snode2 <none> <none>
openebs openebs-ndm-tdj44 1/1 Running 2 20h 192.168.149.5 k8snode1 <none> <none>
openebs openebs-provisioner-7f86c6bb64-4kdwp 1/1 Running 4 20h 10.244.1.21 k8snode2 <none> <none>
openebs openebs-snapshot-operator-54b9c886bf-jghj7 2/2 Running 2 20h 10.244.0.37 k8smaster2 <none> <none>
发现coredns成功运行了，尝试用nslookup命令验证coredns是否正常
# nslookup redis.kubesphere-system.svc
Server: 10.96.0.10
Address: 10.96.0.10#53
Name: redis.kubesphere-system.svc.cluster.local
Address: 10.96.3.171
测试后，发现pod能dns解析其他服务，奇怪的是，现在ks-account的pod状态自动变为running了，查看ks-account的pod描述
[root@k8sMaster2 test]# kubectl describe pods ks-account-75cb9794bd-wpgvf -n kubesphere-system
Name: ks-account-75cb9794bd-wpgvf
Namespace: kubesphere-system
Priority: 0
Node: k8smaster2/192.168.149.4
Start Time: Tue, 12 Nov 2024 04:20:03 +0800
Labels: app=ks-account
pod-template-hash=75cb9794bd
tier=backend
version=v2.1.1
Annotations: kubectl.kubernetes.io/restartedAt: 2024-11-11T20:20:03Z
Status: Running
IP: 10.244.0.101
IPs:
IP: 10.244.0.101
Controlled By: ReplicaSet/ks-account-75cb9794bd
Init Containers:
wait-redis:
Container ID: docker://6ec3aabbc35d7984ff89eefeb174cbb23717539051b6455f5c81967599884cd3
Image: alpine:3.10.4
Image ID: docker-pullable://alpine@sha256:7c3773f7bcc969f03f8f653910001d99a9d324b4b9caa008846ad2c3089f5a5f
Port: <none>
Host Port: <none>
Command:
sh
-c
until nc -z redis.kubesphere-system.svc 6379; do echo "waiting for redis"; sleep 2; done;
State: Terminated
Reason: Completed
Exit Code: 0
Started: Tue, 12 Nov 2024 04:20:04 +0800
Finished: Tue, 12 Nov 2024 04:20:46 +0800
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from kubesphere-token-jpqw6 (ro)
wait-ldap:
Container ID: docker://12ea9eb098d1c54d9649bde40c751deee027e1608fd5bbd3f9561bbb65dbb457
Image: alpine:3.10.4
Image ID: docker-pullable://alpine@sha256:7c3773f7bcc969f03f8f653910001d99a9d324b4b9caa008846ad2c3089f5a5f
Port: <none>
Host Port: <none>
Command:
sh
-c
until nc -z openldap.kubesphere-system.svc 389; do echo "waiting for ldap"; sleep 2; done;
State: Terminated
Reason: Completed
Exit Code: 0
Started: Tue, 12 Nov 2024 04:20:47 +0800
Finished: Tue, 12 Nov 2024 04:20:47 +0800
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from kubesphere-token-jpqw6 (ro)
Containers:
ks-account:
Container ID: docker://bfb19e859084672fd8f78fde1b92b1eb4fc05fec9541905ab205b71b60a786a9
Image: kubesphere/ks-account:v2.1.1
Image ID: docker-pullable://kubesphere/ks-account@sha256:6fccef53ab7a269160ce7816dfe3583730ac7fe2064ea5c9e3ce5e366f3470eb
Port: 9090/TCP
Host Port: 0/TCP
Command:
ks-iam
--logtostderr=true
--jwt-secret=$(JWT_SECRET)
--admin-password=$(ADMIN_PASSWORD)
--enable-multi-login=False
--token-idle-timeout=40m
--redis-url=redis://redis.kubesphere-system.svc:6379
--generate-kubeconfig=true
State: Running
Started: Tue, 12 Nov 2024 04:36:55 +0800
Last State: Terminated
Reason: Error
Exit Code: 2
Started: Tue, 12 Nov 2024 04:31:20 +0800
Finished: Tue, 12 Nov 2024 04:31:46 +0800
Ready: True
Restart Count: 7
Limits:
cpu: 1
memory: 500Mi
Requests:
cpu: 20m
memory: 100Mi
Environment:
KUBECTL_IMAGE: kubesphere/kubectl:v1.0.0
JWT_SECRET: <set to the key 'jwt-secret' in secret 'ks-account-secret'> Optional: false
ADMIN_PASSWORD: <set to the key 'admin-password' in secret 'ks-account-secret'> Optional: false
Mounts:
/etc/ks-iam from user-init (rw)
/etc/kubesphere from kubesphere-config (rw)
/etc/kubesphere/rules from policy-rules (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kubesphere-token-jpqw6 (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
policy-rules:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: policy-rules
Optional: false
user-init:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: user-init
Optional: false
kubesphere-config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: kubesphere-config
Optional: false
kubesphere-token-jpqw6:
Type: Secret (a volume populated by a Secret)
SecretName: kubesphere-token-jpqw6
Optional: false
QoS Class: Burstable
Node-Selectors: <none>
Tolerations: CriticalAddonsOnly
node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 60s
node.kubernetes.io/unreachable:NoExecute for 60s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 32m default-scheduler Successfully assigned kubesphere-system/ks-account-75cb9794bd-wpgvf to k8smaster2
Normal Pulled 32m kubelet, k8smaster2 Container image "alpine:3.10.4" already present on machine
Normal Created 32m kubelet, k8smaster2 Created container wait-redis
Normal Started 32m kubelet, k8smaster2 Started container wait-redis
Normal Pulled 31m kubelet, k8smaster2 Container image "alpine:3.10.4" already present on machine
Normal Created 31m kubelet, k8smaster2 Created container wait-ldap
Normal Started 31m kubelet, k8smaster2 Started container wait-ldap
Normal Created 29m (x4 over 31m) kubelet, k8smaster2 Created container ks-account
Normal Started 29m (x4 over 31m) kubelet, k8smaster2 Started container ks-account
Normal Pulled 27m (x5 over 31m) kubelet, k8smaster2 Container image "kubesphere/ks-account:v2.1.1" already present on machine
Warning BackOff 17m (x43 over 30m) kubelet, k8smaster2 Back-off restarting failed container
查看ks-apigateway的pod描述
[root@k8sMaster2 test]# kubectl describe pods ks-apigateway-79b78f9648-pcgfr -n kubesphere-system
Name: ks-apigateway-79b78f9648-pcgfr
Namespace: kubesphere-system
Priority: 0
Node: k8smaster2/192.168.149.4
Start Time: Tue, 12 Nov 2024 04:20:02 +0800
Labels: app=ks-apigateway
pod-template-hash=79b78f9648
tier=backend
version=v2.1.1
Annotations: kubectl.kubernetes.io/restartedAt: 2024-11-11T20:20:02Z
Status: Running
IP: 10.244.0.100
IPs:
IP: 10.244.0.100
Controlled By: ReplicaSet/ks-apigateway-79b78f9648
Containers:
ks-apigateway:
Container ID: docker://6ef198f71bbabeb51b5f2e1ae1bd9ae24b92f41626049ce53477ded4327557e2
Image: kubesphere/ks-apigateway:v2.1.1
Image ID: docker-pullable://kubesphere/ks-apigateway@sha256:805d1e89aebd391d04bcc0d10fcef2529bad88e0919b5751c043a83fd1781592
Port: 2018/TCP
Host Port: 0/TCP
Command:
/bin/sh
-c
export KUBESPHERE_TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token` && ks-apigateway --conf=/etc/caddy/Caddyfile --log=stderr
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Tue, 12 Nov 2024 04:52:17 +0800
Finished: Tue, 12 Nov 2024 04:52:22 +0800
Ready: False
Restart Count: 11
Limits:
cpu: 1
memory: 500Mi
Requests:
cpu: 20m
memory: 100Mi
Environment:
JWT_SECRET: <set to the key 'jwt-secret' in secret 'ks-account-secret'> Optional: false
Mounts:
/etc/caddy from caddyfile (rw)
/etc/kubesphere from kubesphere-config (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kubesphere-token-jpqw6 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
caddyfile:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: caddyfile
Optional: false
kubesphere-config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: kubesphere-config
Optional: false
kubesphere-token-jpqw6:
Type: Secret (a volume populated by a Secret)
SecretName: kubesphere-token-jpqw6
Optional: false
QoS Class: Burstable
Node-Selectors: <none>
Tolerations: CriticalAddonsOnly
node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 60s
node.kubernetes.io/unreachable:NoExecute for 60s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 34m default-scheduler Successfully assigned kubesphere-system/ks-apigateway-79b78f9648-pcgfr to k8smaster2
Normal Pulled 32m (x5 over 34m) kubelet, k8smaster2 Container image "kubesphere/ks-apigateway:v2.1.1" already present on machine
Normal Created 32m (x5 over 34m) kubelet, k8smaster2 Created container ks-apigateway
Normal Started 32m (x5 over 34m) kubelet, k8smaster2 Started container ks-apigateway
Warning BackOff 4m2s (x134 over 34m) kubelet, k8smaster2 Back-off restarting failed container
