service account是k8s为pod内部的进程访问apiserver创建的一种用户。其实在pod外部也可以通过sa的token和证书访问apiserver，不过在pod外部一般都是采用client 证书的方式。
创建一个namespace，就会自动生成名字为 default 的 service account。
root@master:~# kubectl create ns test
namespace/test created
root@master:~# kubectl get sa -n test
NAME SECRETS AGE
default 1 6s
当然我们也可以再创建额外的sa。
root@master:~# kubectl create sa sa1 -n test
serviceaccount/sa1 created
root@master:~# kubectl get sa -n test
NAME SECRETS AGE
default 1 94s
sa1 1 2s
有了sa后，我们就可以使用sa的token和apiserver交互了，由于所有通信都通过TLS进行，所以也得需要证书（ca.crt，这里的证书指的是server端的ca证书）或者允许不安全的连接(--insecure)。
token和证书如何获取的？每个sa都会自动关联一个secret，token和证书就存在secret中。在pod内部他们被放在如下文件中(所有pod内部的ca.crt证书都一样，都是/etc/kubernetes/pki/ca.crt)
/var/run/secrets/kubernetes.io/serviceaccount/token
/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
在外部可以通过secret获取。下面分别实验这两种方式下如何访问apiserver。
root@master:~# kubectl describe sa sa1 -n test
Name: sa1
Namespace: test
Labels: <none>
Annotations: <none>
Image pull secrets: <none>
Mountable secrets: sa1-token-p5wxt
Tokens: sa1-token-p5wxt
Events: <none>
外部访问apiserver
下面验证在外部如何通过sa的token和证书访问apiserver。
首先获取sa的token，cert和apiserver endpoint。
SERVICE_ACCOUNT=sa1
# Get the ServiceAccount's token Secret's name
SECRET=$(kubectl get serviceaccount -n test ${SERVICE_ACCOUNT} -o json | jq -Mr '.secrets[].name | select(contains("token"))')
# Extract the Bearer token from the Secret and decode
TOKEN=$(kubectl get secret -n test ${SECRET} -o json | jq -Mr '.data.token' | base64 -d)
# Extract, decode and write the ca.crt to a temporary location
kubectl get secret -n test ${SECRET} -o json | jq -Mr '.data["ca.crt"]' | base64 -d > /tmp/ca.crt
# Get the API Server location
APISERVER=$(kubectl config view --minify | grep server | cut -f 2- -d ":" | tr -d " ")
使用curl命令，指定token和insecure(表示不对server端证书进行认证)，开始和apiserver交互。
root@master:~# curl --header "Authorization: Bearer $TOKEN" --insecure -s $APISERVER/api
{
"kind": "APIVersions",
"versions": [
"v1"
],
"serverAddressByClientCIDRs": [
{
"clientCIDR": "0.0.0.0/0",
"serverAddress": "192.168.122.20:6443"
}
]
}root@master:~#
也可以通过 --cacert /tmp/ca.crt 指定证书。
curl的参数--cacert 的作用
(HTTPS) Tells curl to use the specified certificate file to verify the peer. The file may contain multiple CA certificates. The certificate(s) must be in PEM format. If this option is used several times, the last one will be used.
root@master:~# curl --header "Authorization: Bearer $TOKEN" --cacert /tmp/ca.crt -s $APISERVER/api/
{
"kind": "APIVersions",
"versions": [
"v1"
],
"serverAddressByClientCIDRs": [
{
"clientCIDR": "0.0.0.0/0",
"serverAddress": "192.168.122.20:6443"
}
]
}root@master:~#
pod内部访问apiserver
首先创建一个包含curl命令的pod，虽然没有指定sa，但是会自动将test namespace下的default的sa分配给这个pod。
root@master:~# cat <<EOF | kubectl create -f -
> apiVersion: v1
> kind: Pod
> metadata:
> name: test
> namespace: test
>
> spec:
> containers:
> - name: samplepod
> command: ["/bin/sh", "-c", "sleep 99999"]
> image: byrnedo/alpine-curl
> EOF
pod/test created
进入pod内部，获取token，crt。注意的是在pod内部是通过下面的两个环境变量获取apiserver的endpoint的，这里的endpoint是service ip和port，即10.96.0.10:443，而在pod外部使用的endpoint是192.168.122.20:6443.
KUBERNETES_SERVICE_HOST
KUBERNETES_PORT_443_TCP_PORT
root@master:~# kubectl exec -it -n test test sh
获取token和证书
/ # TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token`
/ # APISERVER="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT"
不使用证书访问
/ # curl --header "Authorization: Bearer $TOKEN" --insecure -s $APISERVER/api
{
"kind": "APIVersions",
"versions": [
"v1"
],
"serverAddressByClientCIDRs": [
{
"clientCIDR": "0.0.0.0/0",
"serverAddress": "192.168.122.20:6443"
}
]
}/ #
使用证书访问
/ # CAPATH="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
/ # curl --header "Authorization: Bearer $TOKEN" --cacert $CAPATH -s $APISERVER/api
{
"kind": "APIVersions",
"versions": [
"v1"
],
"serverAddressByClientCIDRs": [
{
"clientCIDR": "0.0.0.0/0",
"serverAddress": "192.168.122.20:6443"
}
]
}/ #
sa的默认权利
当curl请求通过apiserver的认证后，会被分配一个user - system:serviceaccount:test:sa1，和一个group - system:serviceaccounts:test:sa1，同时也会被分配另一个group system:authenticated代表这是一个通过认证的请求。
前面的user和group目前是没有关联任何role或者clusterrole的，这意味着他们是没有任何权利去查看或者修改k8s内部资源的。而system:authenticated是系统自动创建的group，并且已经被默认关联到了下面的三个clusterrole，他们是有查看资源的权利，但是很受限
system:public-info-viewer
system:discovery
system:basic-user
通过下面的clusterrolebinding可看到，上面的三个clusterrole确实绑定到group system:authenticated了。
root@master:~# kubectl describe clusterrolebinding system:public-info-viewer
Name: system:public-info-viewer
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
Role:
Kind: ClusterRole
Name: system:public-info-viewer
Subjects:
Kind Name Namespace
---- ---- ---------
Group system:authenticated
Group system:unauthenticated
root@master:~# kubectl describe clusterrolebinding system:discovery
Name: system:discovery
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
Role:
Kind: ClusterRole
Name: system:discovery
Subjects:
Kind Name Namespace
---- ---- ---------
Group system:authenticated
root@master:~# kubectl describe clusterrolebinding system:basic-user
Name: system:basic-user
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
Role:
Kind: ClusterRole
Name: system:basic-user
Subjects:
Kind Name Namespace
---- ---- ---------
Group system:authenticated
通过下面的命令查看这三个clusterrole都有什么权利，可以看到权利是比较低的，只能查看Non-Resource URLs，不能查看pod，namespace，deployment等资源信息。
root@master:~# kubectl describe clusterrole system:public-info-viewer
Name: system:public-info-viewer
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
[/healthz] [] [get]
[/livez] [] [get]
[/readyz] [] [get]
[/version/] [] [get]
[/version] [] [get]
root@master:~# kubectl describe clusterrole system:discovery
Name: system:discovery
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
[/api/*] [] [get]
[/api] [] [get]
[/apis/*] [] [get]
[/apis] [] [get]
[/healthz] [] [get]
[/livez] [] [get]
[/openapi/*] [] [get]
[/openapi] [] [get]
[/readyz] [] [get]
[/version/] [] [get]
[/version] [] [get]
root@master:~# kubectl describe clusterrole system:basic-user
Name: system:basic-user
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
selfsubjectaccessreviews.authorization.k8s.io [] [] [create]
selfsubjectrulesreviews.authorization.k8s.io [] [] [create]
尝试获取pod信息，但是被Forbidden，因为没有被授权。
root@master:~# curl --header "Authorization: Bearer $TOKEN" --insecure -s $APISERVER/api/v1/namespaces/test
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "namespaces \"test\" is forbidden: User \"system:serviceaccount:test:sa1\" cannot get resource \"namespaces\" in API group \"\" in the namespace \"test\"",
"reason": "Forbidden",
"details": {
"name": "test",
"kind": "namespaces"
},
"code": 403
提高sa权利
如何提高sa的权利呢？
a. 修改默认的这三个clusterrole，但是这是公共的，不建议修改。
b. 将sa绑定的其他权利比较高的clusterrole，比如cluster-admin。
c. 新创建一个role或者clusterrole，指定好需要的权利，将sa绑定上即可。这是推荐的做法。
下面采用第三种方法进行验证。
在test namespace创建一个role read-pod，这个role的权利只可以获取namespace test下的pod。
root@master:~# cat <<EOF | kubectl create -f -
> apiVersion: rbac.authorization.k8s.io/v1
> kind: Role
> metadata:
> namespace: test
> name: read-pod
> rules:
> - apiGroups: [""]
> resources: ["pods"]
> verbs: ["get", "list"]
> EOF
role.rbac.authorization.k8s.io/read-pod created
root@master:~# kubectl create rolebinding test -n test --role read-pod --serviceaccount test:sa1
rolebinding.rbac.authorization.k8s.io/test created
验证一下，可以获取test namespace下的pod
root@master:~# curl --header "Authorization: Bearer $TOKEN" --cacert /tmp/ca.crt -s $APISERVER/api/v1/namespaces/test/pods/test
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "test",
"namespace": "test",
"selfLink": "/api/v1/namespaces/test/pods/test",
"uid": "12ef72cf-be59-4329-8e67-2f3c805a553f",
"resourceVersion": "13801401",
"creationTimestamp": "2020-08-22T22:50:22Z",
"annotations": {
"cni.projectcalico.org/podIP": "10.24.166.144/32",
"cni.projectcalico.org/podIPs": "10.24.166.144/32",
"k8s.v1.cni.cncf.io/network-status": "[{\n \"name\": \"k8s-pod-network\",\n \"ips\": [\n \"10.24.166.144\"\n ],\n \"default\": true,\n \"dns\": {}\n}]",
"k8s.v1.cni.cncf.io/networks-status": "[{\n \"name\": \"k8s-pod-network\",\n \"ips\": [\n \"10.24.166.144\"\n ],\n \"default\": true,\n \"dns\": {}\n}]"
}
},
...
但是pod的子资源是不能获取的，比如获取pods/logs,因为role里只指定了pod资源。如果想获取子资源，还得单独指定。
root@master:~# curl --header "Authorization: Bearer $TOKEN" --cacert /tmp/ca.crt -s $APISERVER/api/v1/namespaces/test/pods/test/logs
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "pods \"test\" is forbidden: User \"system:serviceaccount:test:sa1\" cannot get resource \"pods/logs\" in API group \"\" in the namespace \"test\"",
"reason": "Forbidden",
"details": {
"name": "test",
"kind": "pods"
},
"code": 403
Non-resource requests 和 resource requests
下面一段话是官网对这两个概念的解释，但是还是不太明白什么意思。
Non-resource requests Requests to endpoints other than /api/v1/... or /apis/<group>/<version>/... are considered "non-resource requests", and use the lower-cased HTTP method of the request as the verb. For example, a GET request to endpoints like /api or /healthz would use get as the verb.
Resource requests To determine the request verb for a resource API endpoint, review the HTTP verb used and whether or not the request acts on an individual resource or a collection of resources:
而且查看clusterrole时，也把这两种请求区分开来，如下
root@master:~# kubectl describe clusterrole system:discovery
Name: system:discovery
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
[/api/*] [] [get]
[/api] [] [get]
[/apis/*] [] [get]
[/apis] [] [get]
[/healthz] [] [get]
[/livez] [] [get]
[/openapi/*] [] [get]
[/openapi] [] [get]
[/readyz] [] [get]
[/version/] [] [get]
[/version] [] [get]
所以看了下源码，如果请求的url path满足下面的三个条件之一的话就是Non-Resource request，否则就是 resource request。
a. 请求url path字段小于3，比如
/livez(一个字段)，/api(一个字段), /api/v1(两个字段)等
b. 如果请求url path大于等于3了，但是url path不是以 api 或者 apis开始。比如 /livez/poststarthook/crd-informer-synced
c. url path以/apis开始的，但是后面的字段小于3，比如
/apis/{api-group}或者 /apis/{api-group}/{version}
代码路径
./staging/src/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go
如下结构体用于保存解析http请求的内容
// RequestInfo holds information parsed from the http.Request
type RequestInfo struct {
// IsResourceRequest indicates whether or not the request is for an API resource or subresource
IsResourceRequest bool
// Path is the URL path of the request
Path string
// Verb is the kube verb associated with the request for API requests, not the http verb. This includes things like list and watch.
// for non-resource requests, this is the lowercase http verb
Verb string
APIPrefix string
APIGroup string
APIVersion string
Namespace string
// Resource is the name of the resource being requested. This is not the kind. For example: pods
Resource string
// Subresource is the name of the subresource being requested. This is a different resource, scoped to the parent resource, but it may have a different kind.
// For instance, /pods has the resource "pods" and the kind "Pod", while /pods/foo/status has the resource "pods", the sub resource "status", and the kind "Pod"
// (because status operates on pods). The binding resource for a pod though may be /pods/foo/binding, which has resource "pods", subresource "binding", and kind "Binding".
Subresource string
// Name is empty for some verbs, but if the request directly indicates a name (not in body content) then this field is filled in.
Name string
// Parts are the path parts for the request, always starting with /{resource}/{name}
Parts []string
}
func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, error) {
// start with a non-resource request until proven otherwise
requestInfo := RequestInfo{
IsResourceRequest: false,
Path: req.URL.Path,
Verb: strings.ToLower(req.Method),
}
//如果请求的字段小于3，则认为是 no-resource 请求，比如 /healthz，/readyz
currentParts := splitPath(req.URL.Path)
if len(currentParts) < 3 {
// return a non-resource request
return &requestInfo, nil
}
//不是以 api 或者 apis开始的都认为是 no-resource 请求，
if !r.APIPrefixes.Has(currentParts[0]) {
// return a non-resource request
return &requestInfo, nil
}
requestInfo.APIPrefix = currentParts[0]
currentParts = currentParts[1:]
//走到这里说明url path开始是api或者是apis。
//下面的判断是如果不是api开始的，就是说以apis开始的请求。
if !r.GrouplessAPIPrefixes.Has(requestInfo.APIPrefix) {
//apis开始的请求，如果后面的字段小于3，也表示 non-resource 请求，比如 /apis/apiregistration.k8s.io/v1
// one part (APIPrefix) has already been consumed, so this is actually "do we have four parts?"
if len(currentParts) < 3 {
// return a non-resource request
return &requestInfo, nil
}
requestInfo.APIGroup = currentParts[0]
currentParts = currentParts[1:]
}
requestInfo.IsResourceRequest = true
requestInfo.APIVersion = currentParts[0]
currentParts = currentParts[1:]
// handle input of form /{specialVerb}/*
if specialVerbs.Has(currentParts[0]) {
if len(currentParts) < 2 {
return &requestInfo, fmt.Errorf("unable to determine kind and namespace from url, %v", req.URL)
}
requestInfo.Verb = currentParts[0]
currentParts = currentParts[1:]
} else {
switch req.Method {
case "POST":
requestInfo.Verb = "create"
case "GET", "HEAD":
requestInfo.Verb = "get"
case "PUT":
requestInfo.Verb = "update"
case "PATCH":
requestInfo.Verb = "patch"
case "DELETE":
requestInfo.Verb = "delete"
default:
requestInfo.Verb = ""
}
}
// URL forms: /namespaces/{namespace}/{kind}/*, where parts are adjusted to be relative to kind
if currentParts[0] == "namespaces" {
if len(currentParts) > 1 {
requestInfo.Namespace = currentParts[1]
// if there is another step after the namespace name and it is not a known namespace subresource
// move currentParts to include it as a resource in its own right
if len(currentParts) > 2 && !namespaceSubresources.Has(currentParts[2]) {
currentParts = currentParts[2:]
}
}
} else {
requestInfo.Namespace = metav1.NamespaceNone
}
// parsing successful, so we now know the proper value for .Parts
requestInfo.Parts = currentParts
// parts look like: resource/resourceName/subresource/other/stuff/we/don't/interpret
switch {
case len(requestInfo.Parts) >= 3 && !specialVerbsNoSubresources.Has(requestInfo.Verb):
requestInfo.Subresource = requestInfo.Parts[2]
fallthrough
case len(requestInfo.Parts) >= 2:
requestInfo.Name = requestInfo.Parts[1]
fallthrough
case len(requestInfo.Parts) >= 1:
requestInfo.Resource = requestInfo.Parts[0]
}
// if there's no name on the request and we thought it was a get before, then the actual verb is a list or a watch
if len(requestInfo.Name) == 0 && requestInfo.Verb == "get" {
opts := metainternalversion.ListOptions{}
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), metav1.SchemeGroupVersion, &opts); err != nil {
// An error in parsing request will result in default to "list" and not setting "name" field.
klog.Errorf("Couldn't parse request %#v: %v", req.URL.Query(), err)
// Reset opts to not rely on partial results from parsing.
// However, if watch is set, let's report it.
opts = metainternalversion.ListOptions{}
if values := req.URL.Query()["watch"]; len(values) > 0 {
switch strings.ToLower(values[0]) {
case "false", "0":
default:
opts.Watch = true
}
}
}
if opts.Watch {
requestInfo.Verb = "watch"
} else {
requestInfo.Verb = "list"
}
if opts.FieldSelector != nil {
if name, ok := opts.FieldSelector.RequiresExactMatch("metadata.name"); ok {
if len(path.IsValidPathSegmentName(name)) == 0 {
requestInfo.Name = name
}
}
}
}
// if there's no name on the request and we thought it was a delete before, then the actual verb is deletecollection
if len(requestInfo.Name) == 0 && requestInfo.Verb == "delete" {
requestInfo.Verb = "deletecollection"
}
return &requestInfo, nil
}
从上述源码也能看出 http verb如何转换成 request verb
POST -> create
GET/HEAD with resourceName -> get
GET/HEAD without resourceName -> list(如果没有指定资源名字，则列出所有的资源，比如指定了获取pod1 /api/v1/namespaces/test/pods/pod1，则只获取pod1的信息，如果不指定pod名字，就会返回test namespace下的所有pod)
PUT-> update
PATCH->patch
DELETE with resourceName ->delete
DELETE without resourceName ->delete(同get，如果没有指定删除具体的资源，则删除所有的资源)
Non-resource requests只能在clusterrole中配置，而resource requests可以在role或者clusterrole中配置。
Non-resource requests 和resource requests的配置格式也不一样，如下
//resource requests
rules:
- apiGroups: [""]
#
# at the HTTP level, the name of the resource for accessing Node
# objects is "nodes"
resources: ["nodes"]
verbs: ["get", "list", "watch"]
//Non-resource requests
rules:
- nonResourceURLs: ["/healthz", "/healthz/*"] # '*' in a nonResourceURL is a suffix glob match
verbs: ["get", "post"]
参考
service account相关
https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/
授权相关
https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/
https://kubernetes.io/docs/reference/access-authn-authz/rbac/
https://kubernetes.io/docs/reference/access-authn-authz/authorization/
https://kubernetes.io/docs/reference/access-authn-authz/authentication/
